more condensed stats summary output (closes #3)

logwatch.py

@@ -170,6 +170,81 @@ class Configuration(import_logs.Configuration):
                     self._sanitize_prefixed_output_path(os.path.join(p, prefix_path))
 
 
+class Statistics(import_logs.Statistics):
+    
+    def print_summary(self):
+        
+        invalid_lines_summary = ''
+        if self.invalid_lines:
+            invalid_lines_summary = '\ninvalid lines (malformed/erroneous tracker):\n%s' % textwrap.fill(", ".join(self.invalid_lines), 80, initial_indent="    ", subsequent_indent="    ")
+
+        sites_created_summary = ''
+        if len(self.matomo_sites_created):
+            sites_created_summary = '\n' + self._indent_text(
+                ['%s (ID: %d)' % (hostname, site_id) for hostname, site_id in self.matomo_sites_created],
+                level=2,
+            )
+            
+        sites_ignored_summary = ''
+        if len(self.matomo_sites_ignored):
+            sites_ignored_summary = '\n' + self._indent_text(
+                    self.matomo_sites_ignored, level=2,
+                )
+
+        summary = '''batch import statistics:
+- imported: %(count_lines_recorded)d, downloads: %(count_lines_downloads)d
+- ignored: %(total_lines_ignored)d
+  - HTTP errors: %(count_lines_skipped_http_errors)d, redirects: %(count_lines_skipped_http_redirects)d
+  - invalid: %(count_lines_invalid)d, filtered: %(count_lines_filtered)d
+  - no site match: %(count_lines_no_site)d, no hostname match: %(count_lines_hostname_skipped)d
+  - bots or search engines: %(count_lines_skipped_user_agent)d
+  - static: %(count_lines_static)d, downloads not matching any download extensions %(count_lines_skipped_downloads)d%(invalid_lines)s
+- imported to: %(total_sites)d sites
+  - existing sites: %(total_sites_existing)d
+  - newly created sites: %(total_sites_created)d%(sites_created)s
+  - non-matching hostnames: %(total_sites_ignored)d%(sites_ignored)s
+- time: %(total_time)ds, %(speed_recording)s reqs/second''' %   {
+            'count_lines_recorded': self.count_lines_recorded.value,
+            'count_lines_downloads': self.count_lines_downloads.value,
+            'total_lines_ignored': sum([
+                    self.count_lines_invalid.value,
+                    self.count_lines_filtered.value,
+                    self.count_lines_skipped_user_agent.value,
+                    self.count_lines_skipped_http_errors.value,
+                    self.count_lines_skipped_http_redirects.value,
+                    self.count_lines_static.value,
+                    self.count_lines_skipped_downloads.value,
+                    self.count_lines_no_site.value,
+                    self.count_lines_hostname_skipped.value,
+                ]),
+            'count_lines_invalid': self.count_lines_invalid.value,
+            'count_lines_filtered': self.count_lines_filtered.value,
+            'count_lines_skipped_user_agent': self.count_lines_skipped_user_agent.value,
+            'count_lines_skipped_http_errors': self.count_lines_skipped_http_errors.value,
+            'count_lines_skipped_http_redirects': self.count_lines_skipped_http_redirects.value,
+            'count_lines_static': self.count_lines_static.value,
+            'count_lines_skipped_downloads': self.count_lines_skipped_downloads.value,
+            'count_lines_no_site': self.count_lines_no_site.value,
+            'count_lines_hostname_skipped': self.count_lines_hostname_skipped.value,
+            'total_sites': len(self.matomo_sites),
+            'total_sites_existing': len(self.matomo_sites - set(site_id for hostname, site_id in self.matomo_sites_created)),
+            'total_sites_created': len(self.matomo_sites_created),
+            'sites_created': sites_created_summary,
+            'total_sites_ignored': len(self.matomo_sites_ignored),
+            'sites_ignored': sites_ignored_summary,
+            'total_time': self.time_stop - self.time_start,
+            'speed_recording': self._round_value(self._compute_speed(
+                    self.count_lines_recorded.value,
+                    self.time_start, self.time_stop,
+                )),
+            'url': config.options.matomo_api_url,
+            'invalid_lines': invalid_lines_summary
+        }
+        
+        for sline in summary.split('\n'):
+            logging.info(sline)
+
+
 # monkey-patching fatal_error() so that we can handle ingestion errors gracefully
 # 
 # first, define a replacement
@@ -289,7 +364,7 @@ while True:
         logging.info("ingesting logfiles...")
         
         # we want stats
-        stats = import_logs.Statistics()
+        stats = Statistics()
         import_logs.stats = stats
         stats.set_time_start()
         if config.options.show_progress: