WIP: implemented --delete-ingested and --delete-failed (closes #5)

logwatch.py

@@ -5,6 +5,7 @@ from inotify_simple import INotify, flags
 import import_logs
 from optparse import OptionParser
 import sys
+import os
 import os.path
 from glob import glob
 from enum import Enum
@@ -52,6 +53,14 @@ class Configuration(import_logs.Configuration):
             "or moved anymore and can be safely ingested."
         )
         
+        self.parser.add_option(
+            '--delete-ingested',
+            dest='delete_ingested',
+            action='store_true',
+            default=False,
+            help="Delete ingested logfiles."
+        )
+        
         self.parser.add_option(
             '--prefix-ingested',
             dest='prefix_ingested',
@@ -60,7 +69,8 @@ class Configuration(import_logs.Configuration):
             "which case it should contain '/'), and is then relative to the directory a given "
             "logfile was originally in: when watching several directories, a prefix of "
             "'ingested/' will place ingested files in './ingested/' subdirectories of "
-            "respective watched directories. Directories will be created if needed."
+            "respective watched directories. Directories will be created if needed. This option "
+            "is ignored if --delete-ingested is used."
         )
         
         self.parser.add_option(
@@ -68,6 +78,7 @@ class Configuration(import_logs.Configuration):
             dest='suffix_ingested',
             default=".ingested",
             help="Rename ingested logfiles using this suffix; it cannot contain any '/' characters."
+            "This option is ignored if --delete-ingested is used."
         )
         
         self.parser.add_option(
@@ -78,6 +89,15 @@ class Configuration(import_logs.Configuration):
             help="Exit when ingestion errors are encountered."
         )
         
+        self.parser.add_option(
+            '--delete-failed',
+            dest='delete_failed',
+            action='store_true',
+            default=False,
+            help="Delete logfiles which failed to be ingsted."
+        )
+        
+        
         self.parser.add_option(
             '--prefix-failed',
             dest='prefix_failed',
@@ -88,7 +108,8 @@ class Configuration(import_logs.Configuration):
             "directories, a prefix of 'failed/' will place such files in './failed/' "
             "subdirectories of respective watched directories. Directories will be created "
             "if needed. This prefix will also be used for files containing information "
-            "on what error was encountered and at which line."
+            "on what error was encountered and at which line. This option is ignored "
+            "if --delete-failed is used."
         )
         
         self.parser.add_option(
@@ -96,7 +117,7 @@ class Configuration(import_logs.Configuration):
             dest='suffix_failed',
             default=".failed",
             help="Rename logfiles that failed to be ingested using this suffix; it cannot "
-            "contain any '/' characters."
+            "contain any '/' characters. This option is ignored if --delete-failed is used."
         )
         
         self.parser.add_option(
@@ -129,8 +150,16 @@ class Configuration(import_logs.Configuration):
         # 
         # either prefix does not have to be a directory and does not have to end in a directory
         #
-        # is the prefix an absolute path?
-        for prefix_path in [self.options.prefix_ingested, self.options.prefix_failed]:
+        # we don't have to sanitize relevant paths if --delete-ingested or --delete-failed are used
+        prefix_paths_to_sanitize = []
+        if not self.options.delete_ingested:
+            prefix_paths_to_sanitize += [self.options.prefix_ingested]
+        if not self.options.delete_failed:
+            prefix_paths_to_sanitize += [self.options.prefix_failed]
+        logging.debug("sanitizing prefix directories: %s" % ', '.join(prefix_paths_to_sanitize))
+        # sanitize the paths    
+        for prefix_path in prefix_paths_to_sanitize:
+            # is the prefix an absolute path?
             if os.path.isabs(prefix_path):
                 self._sanitize_prefixed_output_path(prefix_path)
             # not absolute, we need to handle this for every watched directory
@@ -285,31 +314,45 @@ while True:
                 errmsg, filename, lineno = e.args
                 for msg in errmsg.split('\n'):
                     logging.error(msg)
-                # move the failed file to the location specified by
-                # applying the prefix_failed and suffix_failed, as configured
-                f_dir, f_base = os.path.split(f)
-                f_base += config.options.suffix_failed
-                if os.path.isabs(config.options.prefix_failed):
-                    # not using os.path.join() here since prefix does not have to end in a directory
-                    new_f = config.options.prefix_failed + f_base
+                # do we want to delete or move/rename the file?
+                if config.options.delete_failed:
+                    # delete
+                    logging.warning("ingesting the file failed, deleting")
+                    os.remove(f)
+                # nope, we want to move/rename it
                 else:
-                    new_f = os.path.join(f_dir, config.options.prefix_failed + f_base)
-                logging.warning("ingesting the file failed, moving it to:")
-                logging.warning("- %s" % new_f)
-                shutil.move(f, new_f)
+                    # move the failed file to the location specified by
+                    # applying the prefix_failed and suffix_failed, as configured
+                    f_dir, f_base = os.path.split(f)
+                    f_base += config.options.suffix_failed
+                    if os.path.isabs(config.options.prefix_failed):
+                        # not using os.path.join() here since prefix does not have to end in a directory
+                        new_f = config.options.prefix_failed + f_base
+                    else:
+                        new_f = os.path.join(f_dir, config.options.prefix_failed + f_base)
+                    logging.warning("ingesting the file failed, moving it to:")
+                    logging.warning("- %s" % new_f)
+                    shutil.move(f, new_f)
                 
             else:
-                # once the file is processed, we need to move it to the location specified by
-                # applying the prefix_ingested and suffix_ingested, as configured
-                f_dir, f_base = os.path.split(f)
-                f_base += config.options.suffix_ingested
-                if os.path.isabs(config.options.prefix_ingested):
-                    # not using os.path.join() here since prefix does not have to end in a directory
-                    new_f = config.options.prefix_ingested + f_base
+                # do we want to delete or move/rename the file?
+                if config.options.delete_ingested:
+                    # delete
+                    logging.info("   ingested, deleting")
+                    os.remove(f)
+                # nope, we want to move/rename it
                 else:
-                    new_f = os.path.join(f_dir, config.options.prefix_ingested + f_base)
-                logging.info("   ingested, moving to: %s" % new_f)
-                shutil.move(f, new_f)
+                    # once the file is processed, we need to move it to the location specified by
+                    # applying the prefix_ingested and suffix_ingested, as configured
+                    f_dir, f_base = os.path.split(f)
+                    f_base += config.options.suffix_ingested
+                    if os.path.isabs(config.options.prefix_ingested):
+                        # not using os.path.join() here since prefix does not have to end in a directory
+                        new_f = config.options.prefix_ingested + f_base
+                    else:
+                        new_f = os.path.join(f_dir, config.options.prefix_ingested + f_base)
+                    logging.info("   ingested, moving to: %s" % new_f)
+                    shutil.move(f, new_f)
                 
         # done with stats
         stats.set_time_stop()