Commit f4187ce2 authored by Michał Woźniak's avatar Michał Woźniak

WIP: implemented --delete-ingested and --delete-failed (closes #5)

parent e356af64
......@@ -5,6 +5,7 @@ from inotify_simple import INotify, flags
import import_logs
from optparse import OptionParser
import sys
import os
import os.path
from glob import glob
from enum import Enum
......@@ -52,6 +53,14 @@ class Configuration(import_logs.Configuration):
"or moved anymore and can be safely ingested."
)
self.parser.add_option(
'--delete-ingested',
dest='delete_ingested',
action='store_true',
default=False,
help="Delete ingested logfiles."
)
self.parser.add_option(
'--prefix-ingested',
dest='prefix_ingested',
......@@ -60,7 +69,8 @@ class Configuration(import_logs.Configuration):
"which case it should contain '/'), and is then relative to the directory a given "
"logfile was originally in: when watching several directories, a prefix of "
"'ingested/' will place ingested files in './ingested/' subdirectories of "
"respective watched directories. Directories will be created if needed."
"respective watched directories. Directories will be created if needed. This option "
"is ignored if --delete-ingested is used."
)
self.parser.add_option(
......@@ -68,6 +78,7 @@ class Configuration(import_logs.Configuration):
dest='suffix_ingested',
default=".ingested",
help="Rename ingested logfiles using this suffix; it cannot contain any '/' characters."
"This option is ignored if --delete-ingested is used."
)
self.parser.add_option(
......@@ -78,6 +89,15 @@ class Configuration(import_logs.Configuration):
help="Exit when ingestion errors are encountered."
)
self.parser.add_option(
'--delete-failed',
dest='delete_failed',
action='store_true',
default=False,
help="Delete logfiles which failed to be ingsted."
)
self.parser.add_option(
'--prefix-failed',
dest='prefix_failed',
......@@ -88,7 +108,8 @@ class Configuration(import_logs.Configuration):
"directories, a prefix of 'failed/' will place such files in './failed/' "
"subdirectories of respective watched directories. Directories will be created "
"if needed. This prefix will also be used for files containing information "
"on what error was encountered and at which line."
"on what error was encountered and at which line. This option is ignored "
"if --delete-failed is used."
)
self.parser.add_option(
......@@ -96,7 +117,7 @@ class Configuration(import_logs.Configuration):
dest='suffix_failed',
default=".failed",
help="Rename logfiles that failed to be ingested using this suffix; it cannot "
"contain any '/' characters."
"contain any '/' characters. This option is ignored if --delete-failed is used."
)
self.parser.add_option(
......@@ -129,8 +150,16 @@ class Configuration(import_logs.Configuration):
#
# either prefix does not have to be a directory and does not have to end in a directory
#
# is the prefix an absolute path?
for prefix_path in [self.options.prefix_ingested, self.options.prefix_failed]:
# we don't have to sanitize relevant paths if --delete-ingested or --delete-failed are used
prefix_paths_to_sanitize = []
if not self.options.delete_ingested:
prefix_paths_to_sanitize += [self.options.prefix_ingested]
if not self.options.delete_failed:
prefix_paths_to_sanitize += [self.options.prefix_failed]
logging.debug("sanitizing prefix directories: %s" % ', '.join(prefix_paths_to_sanitize))
# sanitize the paths
for prefix_path in prefix_paths_to_sanitize:
# is the prefix an absolute path?
if os.path.isabs(prefix_path):
self._sanitize_prefixed_output_path(prefix_path)
# not absolute, we need to handle this for every watched directory
......@@ -285,31 +314,45 @@ while True:
errmsg, filename, lineno = e.args
for msg in errmsg.split('\n'):
logging.error(msg)
# move the failed file to the location specified by
# applying the prefix_failed and suffix_failed, as configured
f_dir, f_base = os.path.split(f)
f_base += config.options.suffix_failed
if os.path.isabs(config.options.prefix_failed):
# not using os.path.join() here since prefix does not have to end in a directory
new_f = config.options.prefix_failed + f_base
# do we want to delete or move/rename the file?
if config.options.delete_failed:
# delete
logging.warning("ingesting the file failed, deleting")
os.remove(f)
# nope, we want to move/rename it
else:
new_f = os.path.join(f_dir, config.options.prefix_failed + f_base)
logging.warning("ingesting the file failed, moving it to:")
logging.warning("- %s" % new_f)
shutil.move(f, new_f)
# move the failed file to the location specified by
# applying the prefix_failed and suffix_failed, as configured
f_dir, f_base = os.path.split(f)
f_base += config.options.suffix_failed
if os.path.isabs(config.options.prefix_failed):
# not using os.path.join() here since prefix does not have to end in a directory
new_f = config.options.prefix_failed + f_base
else:
new_f = os.path.join(f_dir, config.options.prefix_failed + f_base)
logging.warning("ingesting the file failed, moving it to:")
logging.warning("- %s" % new_f)
shutil.move(f, new_f)
else:
# once the file is processed, we need to move it to the location specified by
# applying the prefix_ingested and suffix_ingested, as configured
f_dir, f_base = os.path.split(f)
f_base += config.options.suffix_ingested
if os.path.isabs(config.options.prefix_ingested):
# not using os.path.join() here since prefix does not have to end in a directory
new_f = config.options.prefix_ingested + f_base
# do we want to delete or move/rename the file?
if config.options.delete_ingested:
# delete
logging.info(" ingested, deleting")
os.remove(f)
# nope, we want to move/rename it
else:
new_f = os.path.join(f_dir, config.options.prefix_ingested + f_base)
logging.info(" ingested, moving to: %s" % new_f)
shutil.move(f, new_f)
# once the file is processed, we need to move it to the location specified by
# applying the prefix_ingested and suffix_ingested, as configured
f_dir, f_base = os.path.split(f)
f_base += config.options.suffix_ingested
if os.path.isabs(config.options.prefix_ingested):
# not using os.path.join() here since prefix does not have to end in a directory
new_f = config.options.prefix_ingested + f_base
else:
new_f = os.path.join(f_dir, config.options.prefix_ingested + f_base)
logging.info(" ingested, moving to: %s" % new_f)
shutil.move(f, new_f)
# done with stats
stats.set_time_stop()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment