Commit 6d49d511 authored by Michał Woźniak's avatar Michał Woźniak

WIP: ingestion errors are no longer necessarily fatal; preparing for saving info on failures

parent 59a220d5
......@@ -11,6 +11,10 @@ from enum import Enum
import time
import fnmatch
import shutil
import logging
class IngestionException(Exception):
pass
class Configuration(import_logs.Configuration):
......@@ -66,6 +70,44 @@ class Configuration(import_logs.Configuration):
help="Rename ingested logfiles using this suffix; it cannot contain any '/' characters."
)
self.parser.add_option(
'--exit-on-error',
dest='exit_on_error',
action='store_true',
default=False,
help="Exit when ingestion errors are encountered."
)
self.parser.add_option(
'--prefix-failed',
dest='prefix_failed',
default="failed/",
help="Rename logfiles that failed to be ingested using this prefix; prefix can "
"have directories (in which case it should contain '/'), and is then relative "
"to the directory a given logfile was originally in: when watching several "
"directories, a prefix of 'failed/' will place such files in './failed/' "
"subdirectories of respective watched directories. Directories will be created "
"if needed. This prefix will also be used for files containing information "
"on what error was encountered and at which line."
)
self.parser.add_option(
'--suffix-failed',
dest='suffix_failed',
default=".failed",
help="Rename logfiles that failed to be ingested using this suffix; it cannot "
"contain any '/' characters."
)
self.parser.add_option(
'--suffix-failed-error',
dest='suffix_failed_error',
default=".error",
help="Files containing information on the error encountered when ingesting a "
"logfile that failed to be ingested will be named using this suffix (and the "
"--prefix-failed prefix); it cannot contain any '/' characters."
)
# parse the stuff
self._parse_args(self.parser, argv)
......@@ -98,6 +140,32 @@ class Configuration(import_logs.Configuration):
self._sanitize_ingested_output_path(os.path.join(p, self.options.prefix_ingested))
# monkey-patching fatal_error() so that we can handle ingestion errors gracefully
def non_fatal_error(error, filename=None, lineno=None):
# if we have running Recorders, the error might not be fatal
is_fatal = (len(import_logs.Recorder.recorders) == 0) or config.options.exit_on_error
# build error message; `error` can contain either a string, or an Exception
errmsg = str(error)
# if we have filename and lineno, we should save the status of the file and inform the user
if filename and lineno is not None:
errmsg += "\nIngesting '%s' failed at line %s" % (filename, fileno)
# inform, with level depending on whether or not we stop for errors
if is_fatal:
logging.ciritical(errmsg)
# since we exit_on_error, let's exit
sys.exit(1)
# if not fatal, throw an exception
# (since we do need to get out of the particular block we were called in)
else:
raise IngestionException(errmsg, filename, lineno)
orig_fatal_error = import_logs.fatal_error
import_logs.fatal_error = non_fatal_error
# set up configuration, parse command line args, etc
config = Configuration()
import_logs.config = config
......@@ -218,6 +286,9 @@ while True:
except KeyboardInterrupt:
pass
except IngestionException as e:
errmsg, filename, lineno = e.args
logging.error(errmsg)
setup_watches()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment