Commit b8ac2c80 authored by Michał Woźniak's avatar Michał Woźniak

WIP: ready for a live parse test

parent 2db56249
......@@ -98,8 +98,19 @@ class Configuration(import_logs.Configuration):
self._sanitize_ingested_output_path(os.path.join(p, self.options.prefix_ingested))
# set up configuration, parse command line args, etc
config = Configuration()
# this depends on the Configuration()
matomo = Matomo()
# this depends on Matomo()
config.init_token_auth()
# the rest of init
resolver = config.get_resolver()
parser = Parser()
# files to process
logfiles_busy = [] # this is where newly discovered logfiles are added
logfiles_free = [] # logfiles which we can assume are not busy
......@@ -169,25 +180,48 @@ while True:
inotify.rm_watch(wd[0])
watches = {}
print(" +-- simulating log processing of free logfiles...")
for f in logfiles_free:
print(" - %s" % f)
# processing the logfile (simulation right now)
time.sleep(config.options.ingestion_grace_period / 200)
# once the file is processed, we need to move it to the location specified by
# applying the prefix_ingested and suffix_ingested, as configured
f_dir, f_base = os.path.split(f)
f_base += config.options.suffix_ingested
if os.path.isabs(config.options.prefix_ingested):
new_f = config.options.prefix_ingested + f_base
else:
new_f = os.path.join(f_dir, config.options.prefix_ingested + f_base)
print(" -> %s" % new_f)
shutil.move(f, new_f)
logfiles_free = []
# we want stats
stats = Statistics()
stats.set_time_start()
if config.options.show_progress:
stats.start_monitor()
# launch recorders
recorders = Recorder.launch(config.options.recorders)
print(" +-- ingesting logfiles...")
try:
for f in logfiles_free:
print(" - %s" % f)
# processing the logfile
parser.parse(f)
Recorder.wait_empty()
# once the file is processed, we need to move it to the location specified by
# applying the prefix_ingested and suffix_ingested, as configured
f_dir, f_base = os.path.split(f)
f_base += config.options.suffix_ingested
if os.path.isabs(config.options.prefix_ingested):
new_f = config.options.prefix_ingested + f_base
else:
new_f = os.path.join(f_dir, config.options.prefix_ingested + f_base)
print(" done, moving to: %s" % new_f)
shutil.move(f, new_f)
logfiles_free = []
except KeyboardInterrupt:
pass
setup_watches()
# done with stats
stats.set_time_stop()
if config.options.show_progress:
stats.stop_monitor()
stats.print_summary()
del stats
time.sleep(config.options.ingestion_grace_period / 1000)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment