WIP: ingestion errors are no longer necessarily fatal; preparing for saving info on failures

logwatch.py

@@ -11,6 +11,10 @@ from enum import Enum
 import time
 import fnmatch
 import shutil
+import logging
+
+class IngestionException(Exception):
+    pass
 
 class Configuration(import_logs.Configuration):
     
@@ -66,6 +70,44 @@ class Configuration(import_logs.Configuration):
             help="Rename ingested logfiles using this suffix; it cannot contain any '/' characters."
         )
         
+        self.parser.add_option(
+            '--exit-on-error',
+            dest='exit_on_error',
+            action='store_true',
+            default=False,
+            help="Exit when ingestion errors are encountered."
+        )
+        
+        self.parser.add_option(
+            '--prefix-failed',
+            dest='prefix_failed',
+            default="failed/",
+            help="Rename logfiles that failed to be ingested using this prefix; prefix can "
+            "have directories (in which case it should contain '/'), and is then relative "
+            "to the directory a given logfile was originally in: when watching several "
+            "directories, a prefix of 'failed/' will place such files in './failed/' "
+            "subdirectories of respective watched directories. Directories will be created "
+            "if needed. This prefix will also be used for files containing information "
+            "on what error was encountered and at which line."
+        )
+        
+        self.parser.add_option(
+            '--suffix-failed',
+            dest='suffix_failed',
+            default=".failed",
+            help="Rename logfiles that failed to be ingested using this suffix; it cannot "
+            "contain any '/' characters."
+        )
+        
+        self.parser.add_option(
+            '--suffix-failed-error',
+            dest='suffix_failed_error',
+            default=".error",
+            help="Files containing information on the error encountered when ingesting a "
+            "logfile that failed to be ingested will be named using this suffix (and the "
+            "--prefix-failed prefix); it cannot contain any '/' characters."
+        )
+        
         # parse the stuff
         self._parse_args(self.parser, argv)
         
@@ -98,6 +140,32 @@ class Configuration(import_logs.Configuration):
                 self._sanitize_ingested_output_path(os.path.join(p, self.options.prefix_ingested))
 
 
+# monkey-patching fatal_error() so that we can handle ingestion errors gracefully
+def non_fatal_error(error, filename=None, lineno=None):
+
+    # if we have running Recorders, the error might not be fatal
+    is_fatal = (len(import_logs.Recorder.recorders) == 0) or config.options.exit_on_error
+    
+    # build error message; `error` can contain either a string, or an Exception
+    errmsg = str(error)
+    
+    # if we have filename and lineno, we should save the status of the file and inform the user
+    if filename and lineno is not None:
+        errmsg += "\nIngesting '%s' failed at line %s" % (filename, fileno)
+        
+    # inform, with level depending on whether or not we stop for errors
+    if is_fatal:
+        logging.ciritical(errmsg)
+        # since we exit_on_error, let's exit
+        sys.exit(1)
+    # if not fatal, throw an exception
+    # (since we do need to get out of the particular block we were called in)
+    else:
+        raise IngestionException(errmsg, filename, lineno)
+        
+orig_fatal_error = import_logs.fatal_error
+import_logs.fatal_error = non_fatal_error
+
 # set up configuration, parse command line args, etc
 config = Configuration()
 import_logs.config = config
@@ -218,6 +286,9 @@ while True:
         
         except KeyboardInterrupt:
             pass
+        except IngestionException as e:
+            errmsg, filename, lineno = e.args
+            logging.error(errmsg)
         
         setup_watches()