WIP: preparing prefix/suffix handling

logwatch.py

@@ -10,21 +10,43 @@ from glob import glob
 from enum import Enum
 import time
 import fnmatch
+import shutil
 
 class Configuration(import_logs.Configuration):
     
     def __init__(self, argv = None):
+        
         # in case we want to add options
         self.parser = self._create_parser()
+        
         self.parser.add_option(
-            '--ingestion-start-delay',
-            dest='ingestion_start_delay',
+            '--ingestion-grace-period',
+            dest='ingestion_grace_period',
             type='int',
             default=1000,
             help="Delay (in ms) between noticing a logfile to be processed and starting ingesting it."
             "This is part of the built-in heuristic for determining that a file is not being modified "
             "or moved anymore and can be safely ingested."
         )
+        
+        self.parser.add_option(
+            '--prefix-ingested',
+            dest='prefix_ingested',
+            default="ingested/",
+            help="Rename ingested logfiles using this prefix; prefix can have directories (in "
+            "which case it should contain '/'), and is then relative to the directory a given "
+            "logfile was originally in: when watching several directories, a prefix of "
+            "'ingested/' will place ingested files in './ingested/' subdirectories of "
+            "respective watched directories. Directories will be created if needed."
+        )
+        
+        self.parser.add_option(
+            '--suffix-ingested',
+            dest='suffix_ingested',
+            default=".ingested",
+            help="Rename ingested logfiles using this suffix; it cannot contain any '/' characters."
+        )
+        
         # parse the stuff
         self._parse_args(self.parser, argv)
         # sanitize the paths (all paths should be existing directories)
@@ -78,7 +100,7 @@ if len(logfiles_busy) > 0:
 
 # And see the corresponding events:
 timestamp = time.time()
-time.sleep(config.options.ingestion_start_delay / 1000)
+time.sleep(config.options.ingestion_grace_period / 1000)
 while True:
     new_timestamp = time.time()
     print("+-- loop... %0.5fs" % (new_timestamp - timestamp))
@@ -86,7 +108,7 @@ while True:
     logfiles_free = list(set(logfiles_free + logfiles_busy))
     logfiles_busy = []
     
-    for event in inotify.read(config.options.ingestion_start_delay, config.options.ingestion_start_delay):
+    for event in inotify.read(config.options.ingestion_grace_period, config.options.ingestion_grace_period):
         if event.wd not in watches:
             continue
         f = os.path.join(watches[event.wd], event.name)
@@ -110,11 +132,15 @@ while True:
             print("        - %s" % wd[1])
             inotify.rm_watch(wd[0])
         watches = {}
+        
         print("    +-- simulating log processing of free logfiles...")
         for f in logfiles_free:
             print("        - %s" % f)
-            time.sleep(config.options.ingestion_start_delay / 200)
+            
+            # processing the logfile (simulation right now)
+            time.sleep(config.options.ingestion_grace_period / 200)
+            
         logfiles_free = []
         setup_watches()
-        time.sleep(config.options.ingestion_start_delay / 1000)
+        time.sleep(config.options.ingestion_grace_period / 1000)