WIP: ready for a live parse test

logwatch.py

@@ -98,8 +98,19 @@ class Configuration(import_logs.Configuration):
                 self._sanitize_ingested_output_path(os.path.join(p, self.options.prefix_ingested))
 
 
+# set up configuration, parse command line args, etc
 config = Configuration()
 
+# this depends on the Configuration()
+matomo = Matomo()
+
+# this depends on Matomo()
+config.init_token_auth()
+
+# the rest of init
+resolver = config.get_resolver()
+parser = Parser()
+
 # files to process
 logfiles_busy = [] # this is where newly discovered logfiles are added
 logfiles_free = [] # logfiles which we can assume are not busy
@@ -169,25 +180,48 @@ while True:
             inotify.rm_watch(wd[0])
         watches = {}
         
-        print("    +-- simulating log processing of free logfiles...")
-        for f in logfiles_free:
-            print("        - %s" % f)
-            
-            # processing the logfile (simulation right now)
-            time.sleep(config.options.ingestion_grace_period / 200)
-            
-            # once the file is processed, we need to move it to the location specified by
-            # applying the prefix_ingested and suffix_ingested, as configured
-            f_dir, f_base = os.path.split(f)
-            f_base += config.options.suffix_ingested
-            if os.path.isabs(config.options.prefix_ingested):
-                new_f = config.options.prefix_ingested + f_base
-            else:
-                new_f = os.path.join(f_dir, config.options.prefix_ingested + f_base)
-            print("          -> %s" % new_f)
-            shutil.move(f, new_f)
-            
-        logfiles_free = []
+        # we want stats
+        stats = Statistics()
+        stats.set_time_start()
+        if config.options.show_progress:
+            stats.start_monitor()
+        
+        # launch recorders
+        recorders = Recorder.launch(config.options.recorders)
+        
+        print("    +-- ingesting logfiles...")
+        try:
+            for f in logfiles_free:
+                print("        - %s" % f)
+                
+                # processing the logfile
+                parser.parse(f)
+                Recorder.wait_empty()
+                
+                # once the file is processed, we need to move it to the location specified by
+                # applying the prefix_ingested and suffix_ingested, as configured
+                f_dir, f_base = os.path.split(f)
+                f_base += config.options.suffix_ingested
+                if os.path.isabs(config.options.prefix_ingested):
+                    new_f = config.options.prefix_ingested + f_base
+                else:
+                    new_f = os.path.join(f_dir, config.options.prefix_ingested + f_base)
+                print("          done, moving to: %s" % new_f)
+                shutil.move(f, new_f)
+                
+            logfiles_free = []
+        
+        except KeyboardInterrupt:
+            pass
+        
         setup_watches()
+        
+        # done with stats
+        stats.set_time_stop()
+        if config.options.show_progress:
+            stats.stop_monitor()
+        stats.print_summary()
+        del stats
+        
         time.sleep(config.options.ingestion_grace_period / 1000)