Commit 61e9704e authored by Michał Woźniak's avatar Michał Woźniak

WIP: preparing prefix/suffix handling

parent 579cae8f
......@@ -10,21 +10,43 @@ from glob import glob
from enum import Enum
import time
import fnmatch
import shutil
class Configuration(import_logs.Configuration):
def __init__(self, argv = None):
# in case we want to add options
self.parser = self._create_parser()
self.parser.add_option(
'--ingestion-start-delay',
dest='ingestion_start_delay',
'--ingestion-grace-period',
dest='ingestion_grace_period',
type='int',
default=1000,
help="Delay (in ms) between noticing a logfile to be processed and starting ingesting it."
"This is part of the built-in heuristic for determining that a file is not being modified "
"or moved anymore and can be safely ingested."
)
self.parser.add_option(
'--prefix-ingested',
dest='prefix_ingested',
default="ingested/",
help="Rename ingested logfiles using this prefix; prefix can have directories (in "
"which case it should contain '/'), and is then relative to the directory a given "
"logfile was originally in: when watching several directories, a prefix of "
"'ingested/' will place ingested files in './ingested/' subdirectories of "
"respective watched directories. Directories will be created if needed."
)
self.parser.add_option(
'--suffix-ingested',
dest='suffix_ingested',
default=".ingested",
help="Rename ingested logfiles using this suffix; it cannot contain any '/' characters."
)
# parse the stuff
self._parse_args(self.parser, argv)
# sanitize the paths (all paths should be existing directories)
......@@ -78,7 +100,7 @@ if len(logfiles_busy) > 0:
# And see the corresponding events:
timestamp = time.time()
time.sleep(config.options.ingestion_start_delay / 1000)
time.sleep(config.options.ingestion_grace_period / 1000)
while True:
new_timestamp = time.time()
print("+-- loop... %0.5fs" % (new_timestamp - timestamp))
......@@ -86,7 +108,7 @@ while True:
logfiles_free = list(set(logfiles_free + logfiles_busy))
logfiles_busy = []
for event in inotify.read(config.options.ingestion_start_delay, config.options.ingestion_start_delay):
for event in inotify.read(config.options.ingestion_grace_period, config.options.ingestion_grace_period):
if event.wd not in watches:
continue
f = os.path.join(watches[event.wd], event.name)
......@@ -110,11 +132,15 @@ while True:
print(" - %s" % wd[1])
inotify.rm_watch(wd[0])
watches = {}
print(" +-- simulating log processing of free logfiles...")
for f in logfiles_free:
print(" - %s" % f)
time.sleep(config.options.ingestion_start_delay / 200)
# processing the logfile (simulation right now)
time.sleep(config.options.ingestion_grace_period / 200)
logfiles_free = []
setup_watches()
time.sleep(config.options.ingestion_start_delay / 1000)
time.sleep(config.options.ingestion_grace_period / 1000)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment