diff options
author | Chris St. Pierre <chris.a.st.pierre@gmail.com> | 2012-05-18 10:53:35 -0400 |
---|---|---|
committer | Chris St. Pierre <chris.a.st.pierre@gmail.com> | 2012-05-18 16:07:56 -0400 |
commit | bc52f6ac0c56c9551b58fb5110259d87b3a97056 (patch) | |
tree | b0dbe1fa57c47b7cda6d05dfff64b92a6d49973f | |
parent | dbd958c387af890da5e7b455fb409f8e567dce15 (diff) | |
download | bcfg2-bc52f6ac0c56c9551b58fb5110259d87b3a97056.tar.gz bcfg2-bc52f6ac0c56c9551b58fb5110259d87b3a97056.tar.bz2 bcfg2-bc52f6ac0c56c9551b58fb5110259d87b3a97056.zip |
Added inotify filemonitor driver
Moved list of files to ignore into config
-rw-r--r-- | man/bcfg2.conf.5 | 15 | ||||
-rw-r--r-- | src/lib/Bcfg2/Options.py | 4 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Core.py | 20 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor.py | 315 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Fam.py | 82 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Gamin.py | 64 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Inotify.py | 60 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/Pseudo.py | 30 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/FileMonitor/__init__.py | 143 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugin.py | 45 | ||||
-rw-r--r-- | src/lib/Bcfg2/Server/Plugins/Metadata.py | 21 | ||||
-rwxr-xr-x | src/sbin/bcfg2-admin | 1 | ||||
-rwxr-xr-x | src/sbin/bcfg2-info | 1 | ||||
-rwxr-xr-x | src/sbin/bcfg2-lint | 1 | ||||
-rwxr-xr-x | src/sbin/bcfg2-server | 1 | ||||
-rwxr-xr-x | src/sbin/bcfg2-test | 5 |
16 files changed, 449 insertions, 359 deletions
diff --git a/man/bcfg2.conf.5 b/man/bcfg2.conf.5 index 684586892..812a86c76 100644 --- a/man/bcfg2.conf.5 +++ b/man/bcfg2.conf.5 @@ -35,7 +35,13 @@ using the 'bcfg2-admin init' command. .TP .B filemonitor The file monitor used to watch for changes in the repository. -Values of 'gamin', 'fam', or 'pseudo' are valid. +Values of 'inotify', 'gamin', 'fam', or 'pseudo' are valid. The +default is the best available monitor. + +.TP +.B ignore_files +A comma-separated list of globs that should be ignored by the file +monitor. Default: '*~,.#*,*#,*.swp,SCCS,.svn,4913,.gitignore' .TP .B listen_all @@ -133,7 +139,7 @@ administrator supervision is available. (0.9.6 and later) \(bu .B Deps The Deps plugin allows you to make a series of assertions like -"Package X requires Package Y (and optionally also Package Z etc.) +"Package X requires Package Y (and optionally also Package Z etc.)" \(bu .B Editor @@ -441,6 +447,11 @@ sqlite The name of the database to use for statistics data. eg: $REPOSITORY_DIR/etc/bcfg2.sqlite +.SH PLUGIN-SPECIFIC OPTIONS + +Many plugins specify their own options in bcfg2.conf; see the online +documentation about each plugin for more information on these. + .SH SEE ALSO .BR bcfg2(1), .BR bcfg2-server(8) diff --git a/src/lib/Bcfg2/Options.py b/src/lib/Bcfg2/Options.py index 788f67aab..d8a6d4125 100644 --- a/src/lib/Bcfg2/Options.py +++ b/src/lib/Bcfg2/Options.py @@ -308,6 +308,10 @@ SERVER_MCONNECT = Option('Server Metadata Connector list', cook=list_split, cf=('server', 'connectors'), default=['Probes'], ) SERVER_FILEMONITOR = Option('Server file monitor', cf=('server', 'filemonitor'), default='default', odesc='File monitoring driver') +SERVER_FAM_IGNORE = Option('File globs to ignore', + cf=('server', 'ignore_files'), cook=list_split, + default=['*~', '.#*', '*#', '*.swp', 'SCCS', '.svn', + '4913', '.gitignore']) SERVER_LISTEN_ALL = Option('Listen on all interfaces', cf=('server', 'listen_all'), cmd='--listen-all', diff --git a/src/lib/Bcfg2/Server/Core.py b/src/lib/Bcfg2/Server/Core.py index d42c5ad4f..c8ef5b1f7 100644 --- a/src/lib/Bcfg2/Server/Core.py +++ b/src/lib/Bcfg2/Server/Core.py @@ -67,17 +67,28 @@ class Core(Component): filemonitor='default', start_fam_thread=False): Component.__init__(self) self.datastore = repo - if filemonitor not in Bcfg2.Server.FileMonitor.available: + + try: + fm = Bcfg2.Server.FileMonitor.available[filemonitor] + except KeyError: logger.error("File monitor driver %s not available; " "forcing to default" % filemonitor) - filemonitor = 'default' + fm = Bcfg2.Server.FileMonitor.available['default'] + famargs = dict(ignore=[], debug=False) + if 'ignore' in setup: + famargs['ignore'] = setup['ignore'] + if 'debug' in setup: + famargs['debug'] = setup['debug'] + elif 'event debug' in setup: + famargs['debug'] = setup['event debug'] try: - self.fam = Bcfg2.Server.FileMonitor.available[filemonitor]() + self.fam = fm(**famargs) except IOError: logger.error("Failed to instantiate fam driver %s" % filemonitor, exc_info=1) - raise CoreInitError("failed to instantiate fam driver (used %s)" % \ + raise CoreInitError("Failed to instantiate fam driver (used %s)" % filemonitor) + self.pubspace = {} self.cfile = cfile self.cron = {} @@ -198,6 +209,7 @@ class Core(Component): """Shutting down the plugins.""" if not self.terminate.isSet(): self.terminate.set() + self.fam.shutdown() for plugin in list(self.plugins.values()): plugin.shutdown() diff --git a/src/lib/Bcfg2/Server/FileMonitor.py b/src/lib/Bcfg2/Server/FileMonitor.py deleted file mode 100644 index d6b313e6b..000000000 --- a/src/lib/Bcfg2/Server/FileMonitor.py +++ /dev/null @@ -1,315 +0,0 @@ -"""Bcfg2.Server.FileMonitor provides the support for monitorung files.""" - -import logging -import os -import stat -from time import sleep, time - -logger = logging.getLogger('Bcfg2.Server.FileMonitor') - - -def ShouldIgnore(event): - """Test if the event should be suppresed.""" - # FIXME should move event suppression out of the core - if event.filename.split('/')[-1] == '.svn': - return True - if event.filename.endswith('~') or \ - event.filename.startswith('#') or event.filename.startswith('.#'): - #logger.error("Suppressing event for file %s" % (event.filename)) - return True - return False - - -class Event(object): - def __init__(self, request_id, filename, code): - self.requestID = request_id - self.filename = filename - self.action = code - - def code2str(self): - """return static code for event""" - return self.action - -available = {} - - -class FileMonitor(object): - """File Monitor baseclass.""" - def __init__(self, debug=False): - object.__init__(self) - self.debug = debug - self.handles = dict() - - def get_event(self): - return None - - def pending(self): - return False - - def fileno(self): - return 0 - - def handle_one_event(self, event): - if ShouldIgnore(event): - return - if event.requestID not in self.handles: - logger.info("Got event for unexpected id %s, file %s" % - (event.requestID, event.filename)) - return - if self.debug: - logger.info("Dispatching event %s %s to obj %s" \ - % (event.code2str(), event.filename, - self.handles[event.requestID])) - try: - self.handles[event.requestID].HandleEvent(event) - except: - logger.error("error in handling of gamin event for %s" % \ - (event.filename), exc_info=1) - - def handle_event_set(self, lock=None): - count = 1 - event = self.get_event() - start = time() - if lock: - lock.acquire() - try: - self.handle_one_event(event) - while self.pending(): - self.handle_one_event(self.get_event()) - count += 1 - except: - pass - if lock: - lock.release() - end = time() - logger.info("Handled %d events in %.03fs" % (count, (end - start))) - - def handle_events_in_interval(self, interval): - end = time() + interval - while time() < end: - if self.pending(): - self.handle_event_set() - end = time() + interval - else: - sleep(0.5) - - -class FamFam(object): - """The fam object is a set of callbacks for - file alteration events (FAM support). - """ - - def __init__(self): - object.__init__(self) - self.fm = _fam.open() - self.users = {} - self.handles = {} - self.debug = False - - def fileno(self): - """Return fam file handle number.""" - return self.fm.fileno() - - def handle_event_set(self, _): - self.Service() - - def handle_events_in_interval(self, interval): - now = time() - while (time() - now) < interval: - if self.Service(): - now = time() - - def AddMonitor(self, path, obj): - """Add a monitor to path, installing a callback to obj.HandleEvent.""" - mode = os.stat(path)[stat.ST_MODE] - if stat.S_ISDIR(mode): - handle = self.fm.monitorDirectory(path, None) - else: - handle = self.fm.monitorFile(path, None) - self.handles[handle.requestID()] = handle - if obj != None: - self.users[handle.requestID()] = obj - return handle.requestID() - - def Service(self, interval=0.50): - """Handle all fam work.""" - count = 0 - collapsed = 0 - rawevents = [] - start = time() - now = time() - while (time() - now) < interval: - if self.fm.pending(): - while self.fm.pending(): - count += 1 - rawevents.append(self.fm.nextEvent()) - now = time() - unique = [] - bookkeeping = [] - for event in rawevents: - if ShouldIgnore(event): - continue - if event.code2str() != 'changed': - # process all non-change events - unique.append(event) - else: - if (event.filename, event.requestID) not in bookkeeping: - bookkeeping.append((event.filename, event.requestID)) - unique.append(event) - else: - collapsed += 1 - for event in unique: - if event.requestID in self.users: - try: - self.users[event.requestID].HandleEvent(event) - except: - logger.error("handling event for file %s" % (event.filename), exc_info=1) - end = time() - logger.info("Processed %s fam events in %03.03f seconds. %s coalesced" % - (count, (end - start), collapsed)) - return count - - -class Fam(FileMonitor): - """ - The fam object is a set of callbacks for - file alteration events (FAM support). - """ - - def __init__(self, debug=False): - FileMonitor.__init__(self, debug) - self.fm = _fam.open() - - def fileno(self): - return self.fm.fileno() - - def AddMonitor(self, path, obj): - """Add a monitor to path, installing a callback to obj.HandleEvent.""" - mode = os.stat(path)[stat.ST_MODE] - if stat.S_ISDIR(mode): - handle = self.fm.monitorDirectory(path, None) - else: - handle = self.fm.monitorFile(path, None) - if obj != None: - self.handles[handle.requestID()] = obj - return handle.requestID() - - def pending(self): - return self.fm.pending() - - def get_event(self): - return self.fm.nextEvent() - - -class Pseudo(FileMonitor): - """ - The fam object is a set of callbacks for - file alteration events (static monitor support). - """ - - def __init__(self, debug=False): - FileMonitor.__init__(self, debug=False) - self.pending_events = [] - - def pending(self): - return len(self.pending_events) != 0 - - def get_event(self): - return self.pending_events.pop() - - def AddMonitor(self, path, obj): - """add a monitor to path, installing a callback to obj.HandleEvent""" - handleID = len(list(self.handles.keys())) - mode = os.stat(path)[stat.ST_MODE] - handle = Event(handleID, path, 'exists') - if stat.S_ISDIR(mode): - dirList = os.listdir(path) - self.pending_events.append(handle) - for includedFile in dirList: - self.pending_events.append(Event(handleID, - includedFile, - 'exists')) - self.pending_events.append(Event(handleID, path, 'endExist')) - else: - self.pending_events.append(Event(handleID, path, 'exists')) - if obj != None: - self.handles[handleID] = obj - return handleID - - -try: - from gamin import WatchMonitor, GAMCreated, GAMExists, GAMEndExist, \ - GAMChanged, GAMDeleted, GAMMoved - - class GaminEvent(Event): - """ - This class provides an event analogous to - python-fam events based on gamin sources. - """ - def __init__(self, request_id, filename, code): - Event.__init__(self, request_id, filename, code) - action_map = {GAMCreated: 'created', GAMExists: 'exists', - GAMChanged: 'changed', GAMDeleted: 'deleted', - GAMEndExist: 'endExist', GAMMoved: 'moved'} - if code in action_map: - self.action = action_map[code] - - class Gamin(FileMonitor): - """ - The fam object is a set of callbacks for - file alteration events (Gamin support) - """ - def __init__(self, debug=False): - FileMonitor.__init__(self, debug) - self.mon = WatchMonitor() - self.counter = 0 - self.events = [] - - def fileno(self): - return self.mon.get_fd() - - def queue(self, path, action, request_id): - """queue up the event for later handling""" - self.events.append(GaminEvent(request_id, path, action)) - - def AddMonitor(self, path, obj): - """Add a monitor to path, installing a callback to obj.HandleEvent.""" - handle = self.counter - self.counter += 1 - mode = os.stat(path)[stat.ST_MODE] - - # Flush queued gamin events - while self.mon.event_pending(): - self.mon.handle_one_event() - - if stat.S_ISDIR(mode): - self.mon.watch_directory(path, self.queue, handle) - else: - self.mon.watch_file(path, self.queue, handle) - self.handles[handle] = obj - return handle - - def pending(self): - return len(self.events) > 0 or self.mon.event_pending() - - def get_event(self): - if self.mon.event_pending(): - self.mon.handle_one_event() - return self.events.pop(0) - - available['gamin'] = Gamin -except ImportError: - # fall back to _fam - pass - -try: - import _fam - available['fam'] = FamFam -except ImportError: - pass -available['pseudo'] = Pseudo - -for fdrv in ['gamin', 'fam', 'pseudo']: - if fdrv in available: - available['default'] = available[fdrv] - break diff --git a/src/lib/Bcfg2/Server/FileMonitor/Fam.py b/src/lib/Bcfg2/Server/FileMonitor/Fam.py new file mode 100644 index 000000000..1a00fffa0 --- /dev/null +++ b/src/lib/Bcfg2/Server/FileMonitor/Fam.py @@ -0,0 +1,82 @@ +""" Fam provides FAM support for file alteration events """ + +import os +import _fam +import stat +import logging +from time import time +from Bcfg2.Server.FileMonitor import FileMonitor + +logger = logging.getLogger(__name__) + +class Fam(FileMonitor): + __priority__ = 90 + + def __init__(self, ignore=None, debug=False): + FileMonitor.__init__(self, ignore=ignore, debug=debug) + self.fm = _fam.open() + self.users = {} + + def fileno(self): + """Return fam file handle number.""" + return self.fm.fileno() + + def handle_event_set(self, _): + self.Service() + + def handle_events_in_interval(self, interval): + now = time() + while (time() - now) < interval: + if self.Service(): + now = time() + + def AddMonitor(self, path, obj): + """Add a monitor to path, installing a callback to obj.HandleEvent.""" + mode = os.stat(path)[stat.ST_MODE] + if stat.S_ISDIR(mode): + handle = self.fm.monitorDirectory(path, None) + else: + handle = self.fm.monitorFile(path, None) + self.handles[handle.requestID()] = handle + if obj != None: + self.users[handle.requestID()] = obj + return handle.requestID() + + def Service(self, interval=0.50): + """Handle all fam work.""" + count = 0 + collapsed = 0 + rawevents = [] + start = time() + now = time() + while (time() - now) < interval: + if self.fm.pending(): + while self.fm.pending(): + count += 1 + rawevents.append(self.fm.nextEvent()) + now = time() + unique = [] + bookkeeping = [] + for event in rawevents: + if self.should_ignore(event): + continue + if event.code2str() != 'changed': + # process all non-change events + unique.append(event) + else: + if (event.filename, event.requestID) not in bookkeeping: + bookkeeping.append((event.filename, event.requestID)) + unique.append(event) + else: + collapsed += 1 + for event in unique: + if event.requestID in self.users: + try: + self.users[event.requestID].HandleEvent(event) + except: + logger.error("Handling event for file %s" % event.filename, + exc_info=1) + end = time() + logger.info("Processed %s fam events in %03.03f seconds. %s coalesced" % + (count, (end - start), collapsed)) + return count diff --git a/src/lib/Bcfg2/Server/FileMonitor/Gamin.py b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py new file mode 100644 index 000000000..60f80c9c3 --- /dev/null +++ b/src/lib/Bcfg2/Server/FileMonitor/Gamin.py @@ -0,0 +1,64 @@ +""" Gamin driver for file alteration events """ + +import os +import stat +import logging +from gamin import WatchMonitor, GAMCreated, GAMExists, GAMEndExist, \ + GAMChanged, GAMDeleted +from Bcfg2.Server.FileMonitor import Event, FileMonitor + +logger = logging.getLogger(__name__) + +class GaminEvent(Event): + """ + This class provides an event analogous to + python-fam events based on gamin sources. + """ + action_map = {GAMCreated: 'created', GAMExists: 'exists', + GAMChanged: 'changed', GAMDeleted: 'deleted', + GAMEndExist: 'endExist'} + + def __init__(self, request_id, filename, code): + Event.__init__(self, request_id, filename, code) + if code in self.action_map: + self.action = self.action_map[code] + +class Gamin(FileMonitor): + __priority__ = 10 + + def __init__(self, ignore=None, debug=False): + FileMonitor.__init__(self, ignore=ignore, debug=debug) + self.mon = WatchMonitor() + self.counter = 0 + + def fileno(self): + return self.mon.get_fd() + + def queue(self, path, action, request_id): + """queue up the event for later handling""" + self.events.append(GaminEvent(request_id, path, action)) + + def AddMonitor(self, path, obj): + """Add a monitor to path, installing a callback to obj.""" + handle = self.counter + self.counter += 1 + mode = os.stat(path)[stat.ST_MODE] + + # Flush queued gamin events + while self.mon.event_pending(): + self.mon.handle_one_event() + + if stat.S_ISDIR(mode): + self.mon.watch_directory(path, self.queue, handle) + else: + self.mon.watch_file(path, self.queue, handle) + self.handles[handle] = obj + return handle + + def pending(self): + return FileMonitor.pending(self) or self.mon.event_pending() + + def get_event(self): + if self.mon.event_pending(): + self.mon.handle_one_event() + return FileMonitor.get_event(self) diff --git a/src/lib/Bcfg2/Server/FileMonitor/Inotify.py b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py new file mode 100644 index 000000000..9743d868d --- /dev/null +++ b/src/lib/Bcfg2/Server/FileMonitor/Inotify.py @@ -0,0 +1,60 @@ +""" Inotify driver for file alteration events """ + +import os +import stat +import logging +import operator +import pyinotify +from Bcfg2.Server.FileMonitor import FileMonitor, Event + +logger = logging.getLogger(__name__) + +class InotifyEvent(Event): + action_map = {pyinotify.IN_CREATE: 'created', + pyinotify.IN_DELETE: 'deleted', + pyinotify.IN_MODIFY: 'changed'} + + def __init__(self, event): + Event.__init__(self, event.wd, event.pathname, event.mask) + if event.mask in self.action_map: + self.action = self.action_map[event.mask] + + +class Inotify(FileMonitor, pyinotify.ProcessEvent): + __priority__ = 1 + mask = pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MODIFY + + def __init__(self, ignore=None, debug=False): + FileMonitor.__init__(self, ignore=ignore, debug=debug) + self.wm = pyinotify.WatchManager() + self.notifier = pyinotify.ThreadedNotifier(self.wm, self) + self.notifier.start() + + def fileno(self): + return self.wm.get_fd() + + def process_default(self, event): + self.events.append(InotifyEvent(event)) + + def AddMonitor(self, path, obj): + res = self.wm.add_watch(path, self.mask, quiet=False) + if not res: + # if we didn't get a return, but we also didn't get an + # exception, we're already watching this directory, so we + # need to find the watch descriptor for it + for wd, watch in self.wm.watches.items(): + if watch.path == path: + wd = watch.wd + else: + wd = res[path] + self.handles[wd] = obj + self.events.append(Event(wd, path, "exists")) + + mode = os.stat(path)[stat.ST_MODE] + if stat.S_ISDIR(mode): + for wname in os.listdir(path): + self.events.append(Event(wd, wname, "exists")) + return wd + + def shutdown(self): + self.notifier.stop() diff --git a/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py new file mode 100644 index 000000000..4c2d90250 --- /dev/null +++ b/src/lib/Bcfg2/Server/FileMonitor/Pseudo.py @@ -0,0 +1,30 @@ +""" Pseudo provides static monitor support for file alteration events """ + +import os +import stat +import logging +from Bcfg2.Server.FileMonitor import FileMonitor, Event + +logger = logging.getLogger(__name__) + +class Pseudo(FileMonitor): + __priority__ = 99 + + def AddMonitor(self, path, obj): + """add a monitor to path, installing a callback to obj.HandleEvent""" + handleID = len(list(self.handles.keys())) + mode = os.stat(path)[stat.ST_MODE] + handle = Event(handleID, path, 'exists') + if stat.S_ISDIR(mode): + dirList = os.listdir(path) + self.pending_events.append(handle) + for includedFile in dirList: + self.pending_events.append(Event(handleID, + includedFile, + 'exists')) + self.pending_events.append(Event(handleID, path, 'endExist')) + else: + self.pending_events.append(Event(handleID, path, 'exists')) + if obj != None: + self.handles[handleID] = obj + return handleID diff --git a/src/lib/Bcfg2/Server/FileMonitor/__init__.py b/src/lib/Bcfg2/Server/FileMonitor/__init__.py new file mode 100644 index 000000000..40c3253b9 --- /dev/null +++ b/src/lib/Bcfg2/Server/FileMonitor/__init__.py @@ -0,0 +1,143 @@ +"""Bcfg2.Server.FileMonitor provides the support for monitoring files.""" + +import os +import fnmatch +import logging +import pkgutil +from time import sleep, time + +logger = logging.getLogger(__name__) + +class Event(object): + def __init__(self, request_id, filename, code): + self.requestID = request_id + self.filename = filename + self.action = code + + def code2str(self): + """return static code for event""" + return self.action + + def __str__(self): + return "%s: %s %s" % (self.__class__.__name__, + self.filename, self.action) + + def __repr__(self): + return "%s (request ID %s)" % (str(self), self.requestID) + + +class FileMonitor(object): + """File Monitor baseclass.""" + def __init__(self, ignore=None, debug=False): + object.__init__(self) + self.debug = debug + self.handles = dict() + self.events = [] + if ignore is None: + ignore = [] + self.ignore = ignore + + def __str__(self): + return "%s: %s" % (__name__, self.__class__.__name__) + + def __repr__(self): + if self.pending(): + events = "has" + else: + events = "no" + return "%s (%s events, fd %s)" % (str(self), events, self.fileno) + + def should_ignore(self, event): + for pattern in self.ignore: + if (fnmatch.fnmatch(event.filename, pattern) or + fnmatch.fnmatch(os.path.split(event.filename)[-1], pattern)): + if self.debug: + logger.info("Ignoring %s" % event) + return True + return False + + def pending(self): + return bool(self.events) + + def get_event(self): + return self.events.pop(0) + + def fileno(self): + return 0 + + def handle_one_event(self, event): + if self.should_ignore(event): + return + if event.requestID not in self.handles: + logger.info("Got event for unexpected id %s, file %s" % + (event.requestID, event.filename)) + return + if self.debug: + logger.info("Dispatching event %s %s to obj %s" % + (event.code2str(), event.filename, + self.handles[event.requestID])) + try: + self.handles[event.requestID].HandleEvent(event) + except: + logger.error("Error in handling of event for %s" % + event.filename, exc_info=1) + + def handle_event_set(self, lock=None): + count = 1 + event = self.get_event() + start = time() + if lock: + lock.acquire() + try: + self.handle_one_event(event) + while self.pending(): + self.handle_one_event(self.get_event()) + count += 1 + except: + pass + if lock: + lock.release() + end = time() + logger.info("Handled %d events in %.03fs" % (count, (end - start))) + + def handle_events_in_interval(self, interval): + end = time() + interval + while time() < end: + if self.pending(): + self.handle_event_set() + end = time() + interval + else: + sleep(0.5) + + def shutdown(self): + pass + + +available = dict() + +# todo: loading the monitor drivers should be automatic +from Bcfg2.Server.FileMonitor.Pseudo import Pseudo +available['pseudo'] = Pseudo + +try: + from Bcfg2.Server.FileMonitor.Fam import Fam + available['fam'] = Fam +except ImportError: + pass + +try: + from Bcfg2.Server.FileMonitor.Gamin import Gamin + available['gamin'] = Gamin +except ImportError: + pass + +try: + from Bcfg2.Server.FileMonitor.Inotify import Inotify + available['inotify'] = Inotify +except ImportError: + pass + +for fdrv in sorted(available.keys(), key=lambda k: available[k].__priority__): + if fdrv in available: + available['default'] = available[fdrv] + break diff --git a/src/lib/Bcfg2/Server/Plugin.py b/src/lib/Bcfg2/Server/Plugin.py index ca37431a2..90e39e52d 100644 --- a/src/lib/Bcfg2/Server/Plugin.py +++ b/src/lib/Bcfg2/Server/Plugin.py @@ -480,8 +480,8 @@ class DirectoryBacked(object): return if event.requestID not in self.handles: - logger.warn("Got %s event with unknown handle (%s) for %s" - % (action, event.requestID, abspath)) + logger.warn("Got %s event with unknown handle (%s) for %s" % + (action, event.requestID, event.filename)) return # Calculate the absolute and relative paths this event refers to @@ -522,21 +522,13 @@ class DirectoryBacked(object): # didn't know about. Go ahead and treat it like a # "created" event, but log a warning, because this # is unexpected. - logger.warn("Got %s event for unexpected dir %s" % (action, - abspath)) + logger.warn("Got %s event for unexpected dir %s" % + (action, abspath)) self.add_directory_monitor(relpath) else: - logger.warn("Got unknown dir event %s %s %s" % (event.requestID, - event.code2str(), - abspath)) + logger.warn("Got unknown dir event %s %s %s" % + (event.requestID, event.code2str(), abspath)) else: - # Deal with events for non-directories - if ((event.filename[-1] == '~') or - (event.filename[:2] == '.#') or - (event.filename[-4:] == '.swp') or - (event.filename in ['SCCS', '.svn', '4913']) or - (not self.patterns.match(event.filename))): - return if action in ['exists', 'created']: self.add_entry(relpath, event) elif action == 'changed': @@ -547,13 +539,13 @@ class DirectoryBacked(object): # know about. Go ahead and treat it like a # "created" event, but log a warning, because this # is unexpected. - logger.warn("Got %s event for unexpected file %s" % (action, - abspath)) + logger.warn("Got %s event for unexpected file %s" % + (action, + abspath)) self.add_entry(relpath, event) else: - logger.warn("Got unknown file event %s %s %s" % (event.requestID, - event.code2str(), - abspath)) + logger.warn("Got unknown file event %s %s %s" % + (event.requestID, event.code2str(), abspath)) class XMLFileBacked(FileBacked): @@ -583,16 +575,18 @@ class XMLFileBacked(FileBacked): return iter(self.entries) def __str__(self): - return "%s: %s" % (self.name, lxml.etree.tostring(self.xdata)) + return "%s at %s" % (self.__class__.__name__, self.name) class SingleXMLFileBacked(XMLFileBacked): """This object is a coherent cache for an independent XML file.""" - def __init__(self, filename, fam): + def __init__(self, filename, fam, should_monitor=True): XMLFileBacked.__init__(self, filename) self.extras = [] self.fam = fam - self.fam.AddMonitor(filename, self) + self.should_monitor = should_monitor + if should_monitor: + self.fam.AddMonitor(filename, self) def _follow_xincludes(self, fname=None, xdata=None): ''' follow xincludes, adding included files to fam and to @@ -614,8 +608,9 @@ class SingleXMLFileBacked(XMLFileBacked): self._follow_xincludes(fname=fpath) def add_monitor(self, fpath, fname): - self.fam.AddMonitor(fpath, self) - self.extras.append(fname) + if self.should_monitor: + self.fam.AddMonitor(fpath, self) + self.extras.append(fname) def Index(self): """Build local data structures.""" @@ -1207,7 +1202,7 @@ class GroupSpool(Plugin, Generator): name = self.data + relative if relative not in list(self.handles.values()): if not posixpath.isdir(name): - print("Failed to open directory %s" % (name)) + self.logger.error("Failed to open directory %s" % name) return reqid = self.core.fam.AddMonitor(name, self) self.handles[reqid] = relative diff --git a/src/lib/Bcfg2/Server/Plugins/Metadata.py b/src/lib/Bcfg2/Server/Plugins/Metadata.py index 0cb4dc087..50604f5cb 100644 --- a/src/lib/Bcfg2/Server/Plugins/Metadata.py +++ b/src/lib/Bcfg2/Server/Plugins/Metadata.py @@ -38,13 +38,18 @@ class MetadataRuntimeError(Exception): class XMLMetadataConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked): """Handles xml config files and all XInclude statements""" def __init__(self, metadata, watch_clients, basefile): - Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, - os.path.join(metadata.data, - basefile), - metadata.core.fam) + # we tell SingleXMLFileBacked _not_ to add a monitor for this + # file, because the main Metadata plugin has already added + # one. then we immediately set should_monitor to the proper + # value, so that XIinclude'd files get properly watched + fpath = os.path.join(metadata.data, basefile) + Bcfg2.Server.Plugin.SingleXMLFileBacked.__init__(self, fpath, + metadata.core.fam, + should_monitor=False) + self.should_monitor = watch_clients self.metadata = metadata + self.fam = metadata.core.fam self.basefile = basefile - self.should_monitor = watch_clients self.data = None self.basedata = None self.basedir = metadata.data @@ -64,12 +69,6 @@ class XMLMetadataConfig(Bcfg2.Server.Plugin.SingleXMLFileBacked): raise MetadataRuntimeError return self.basedata - def add_monitor(self, fpath, fname): - """Add a fam monitor for an included file""" - if self.should_monitor: - self.metadata.core.fam.AddMonitor(fpath, self.metadata) - self.extras.append(fname) - def load_xml(self): """Load changes from XML""" try: diff --git a/src/sbin/bcfg2-admin b/src/sbin/bcfg2-admin index 007dd0af3..7cc19be8f 100755 --- a/src/sbin/bcfg2-admin +++ b/src/sbin/bcfg2-admin @@ -44,6 +44,7 @@ def main(): 'plugins': Bcfg2.Options.SERVER_PLUGINS, 'event debug': Bcfg2.Options.DEBUG, 'filemonitor': Bcfg2.Options.SERVER_FILEMONITOR, + 'ignore': Bcfg2.Options.SERVER_FAM_IGNORE, 'password': Bcfg2.Options.SERVER_PASSWORD, 'encoding': Bcfg2.Options.ENCODING, } diff --git a/src/sbin/bcfg2-info b/src/sbin/bcfg2-info index 8598a58eb..fdcf9ac17 100755 --- a/src/sbin/bcfg2-info +++ b/src/sbin/bcfg2-info @@ -648,6 +648,7 @@ if __name__ == '__main__': 'password': Bcfg2.Options.SERVER_PASSWORD, 'mconnect': Bcfg2.Options.SERVER_MCONNECT, 'filemonitor': Bcfg2.Options.SERVER_FILEMONITOR, + 'ignore': Bcfg2.Options.SERVER_FAM_IGNORE, 'location': Bcfg2.Options.SERVER_LOCATION, 'static': Bcfg2.Options.SERVER_STATIC, 'key': Bcfg2.Options.SERVER_KEY, diff --git a/src/sbin/bcfg2-lint b/src/sbin/bcfg2-lint index 78b833f02..bc1e5b70e 100755 --- a/src/sbin/bcfg2-lint +++ b/src/sbin/bcfg2-lint @@ -82,6 +82,7 @@ if __name__ == '__main__': 'plugins': Bcfg2.Options.SERVER_PLUGINS, 'mconnect': Bcfg2.Options.SERVER_MCONNECT, 'filemonitor': Bcfg2.Options.SERVER_FILEMONITOR, + 'ignore': Bcfg2.Options.SERVER_FAM_IGNORE, 'location': Bcfg2.Options.SERVER_LOCATION, 'static': Bcfg2.Options.SERVER_STATIC, 'key': Bcfg2.Options.SERVER_KEY, diff --git a/src/sbin/bcfg2-server b/src/sbin/bcfg2-server index 757172464..1b8b0d158 100755 --- a/src/sbin/bcfg2-server +++ b/src/sbin/bcfg2-server @@ -26,6 +26,7 @@ if __name__ == '__main__': 'plugins' : Bcfg2.Options.SERVER_PLUGINS, 'password' : Bcfg2.Options.SERVER_PASSWORD, 'fm' : Bcfg2.Options.SERVER_FILEMONITOR, + 'ignore' : Bcfg2.Options.SERVER_FAM_IGNORE, 'key' : Bcfg2.Options.SERVER_KEY, 'cert' : Bcfg2.Options.SERVER_CERT, 'ca' : Bcfg2.Options.SERVER_CA, diff --git a/src/sbin/bcfg2-test b/src/sbin/bcfg2-test index e3cfd27cc..7ddbb3509 100755 --- a/src/sbin/bcfg2-test +++ b/src/sbin/bcfg2-test @@ -70,7 +70,8 @@ def main(): 'password': Bcfg2.Options.SERVER_PASSWORD, 'verbose': Bcfg2.Options.VERBOSE, 'noseopts': Bcfg2.Options.TEST_NOSEOPTS, - 'ignore': Bcfg2.Options.TEST_IGNORE, + 'ignore': Bcfg2.Options.SERVER_FAM_IGNORE, + 'test_ignore': Bcfg2.Options.TEST_IGNORE, 'validate': Bcfg2.Options.CFG_VALIDATION, } setup = Bcfg2.Options.OptionParser(optinfo) @@ -92,7 +93,7 @@ def main(): ) ignore = dict() - for entry in setup['ignore']: + for entry in setup['test_ignore']: tag, name = entry.split(":") try: ignore[tag].append(name) |