Commit db8e900b authored by Michał Woźniak's avatar Michał Woźniak

WIP: debug/info/warning/error output is... reasonable

parent 5fc86955
......@@ -149,7 +149,8 @@ def non_fatal_error(error, filename=None, lineno=None):
# inform, with level depending on whether or not we stop for errors
if is_fatal:
logging.ciritical(errmsg)
for msg in errmsg.split('\n'):
logging.ciritical(msg)
# since we exit_on_error, let's exit
sys.exit(1)
# if not fatal, throw an exception
......@@ -192,9 +193,9 @@ watches = {}
def setup_watches():
global watches
global logfiles_busy
print(" +-- watching:")
logging.info("watching:")
for path in config.filenames:
print(" - %s" % path)
logging.info("- %s" % path)
# set up watches
wd = inotify.add_watch(path, watch_flags)
watches[wd] = path
......@@ -203,20 +204,20 @@ def setup_watches():
if p not in logfiles_busy:
logfiles_busy.append(p)
print("+-- setting up watches...")
logging.debug("setting up watches...")
setup_watches()
if len(logfiles_busy) > 0:
print("+-- initial logfiles:")
logging.info("found logfiles:")
for f in logfiles_busy:
print(" - %s" % f)
logging.info("- %s" % f)
# And see the corresponding events:
timestamp = time.time()
time.sleep(config.options.ingestion_grace_period / 1000)
while True:
new_timestamp = time.time()
print("+-- loop... %0.5fs" % (new_timestamp - timestamp))
logging.debug("looped in %0.5fs" % (new_timestamp - timestamp))
timestamp = new_timestamp
logfiles_free = list(set(logfiles_free + logfiles_busy))
logfiles_busy = []
......@@ -225,40 +226,43 @@ while True:
if event.wd not in watches:
continue
f = os.path.join(watches[event.wd], event.name)
print(' +-- event! %s: %s' % (f, ', '.join([str(flag) for flag in flags.from_mask(event.mask)])))
logging.debug('- event! %s: %s' % (f, ', '.join([str(flag) for flag in flags.from_mask(event.mask)])))
if f in logfiles_free:
logfiles_free.remove(f)
logfiles_busy.append(f)
print(" - busy...")
logging.debug(" flags the file as busy...")
elif f not in logfiles_busy:
if fnmatch.fnmatch(f, config.options.logfiles_glob):
print(" - new logfile spotted!")
logging.debug(" new logfile spotted!")
logfiles_busy.append(f)
if len(logfiles_free) > 0:
print(" +-- free logfiles:")
logging.info("found logfiles to process:")
for f in logfiles_free:
print(" - %s" % f)
logging.info("- %s" % f)
print(" +-- removing watches")
logging.debug("removing watches")
for wd in watches.items():
print(" - %s" % wd[1])
logging.debug("- %s" % wd[1])
inotify.rm_watch(wd[0])
watches = {}
# inform
logging.info("ingesting logfiles...")
# we want stats
stats = import_logs.Statistics()
import_logs.stats = stats
stats.set_time_start()
if config.options.show_progress:
stats.start_monitor()
# launch recorders
recorders = import_logs.Recorder.launch(config.options.recorders)
print(" +-- ingesting logfiles...")
# ingest files one by one
for f in logfiles_free:
print(" - %s" % f)
logging.info("- %s..." % f)
try:
# processing the logfile
......@@ -270,7 +274,8 @@ while True:
except IngestionException as e:
errmsg, filename, lineno = e.args
logging.error(errmsg)
for msg in errmsg.split('\n'):
logging.error(msg)
# move the failed file to the location specified by
# applying the prefix_failed and suffix_failed, as configured
f_dir, f_base = os.path.split(f)
......@@ -280,7 +285,8 @@ while True:
new_f = config.options.prefix_failed + f_base
else:
new_f = os.path.join(f_dir, config.options.prefix_failed + f_base)
print(" error, moving to: %s" % new_f)
logging.warning("ingesting the file failed, moving it to:")
logging.warning("- %s" % new_f)
shutil.move(f, new_f)
else:
......@@ -293,22 +299,22 @@ while True:
new_f = config.options.prefix_ingested + f_base
else:
new_f = os.path.join(f_dir, config.options.prefix_ingested + f_base)
print(" done, moving to: %s" % new_f)
logging.info(" ingested, moving to: %s" % new_f)
shutil.move(f, new_f)
# clear files to be processed
# all have either been processed, or errored out
logfiles_free = []
# get the watches going again
setup_watches()
# done with stats
stats.set_time_stop()
if config.options.show_progress:
stats.stop_monitor()
stats.print_summary()
del stats
# clear files to be processed
# all have either been processed, or errored out
logfiles_free = []
# get the watches going again
setup_watches()
time.sleep(config.options.ingestion_grace_period / 1000)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment