Commit 0e0e0c75 authored by Michał Woźniak's avatar Michał Woźniak

more condensed stats summary output (closes #3)

parent f4187ce2
......@@ -170,6 +170,81 @@ class Configuration(import_logs.Configuration):
self._sanitize_prefixed_output_path(os.path.join(p, prefix_path))
class Statistics(import_logs.Statistics):
def print_summary(self):
invalid_lines_summary = ''
if self.invalid_lines:
invalid_lines_summary = '\ninvalid lines (malformed/erroneous tracker):\n%s' % textwrap.fill(", ".join(self.invalid_lines), 80, initial_indent=" ", subsequent_indent=" ")
sites_created_summary = ''
if len(self.matomo_sites_created):
sites_created_summary = '\n' + self._indent_text(
['%s (ID: %d)' % (hostname, site_id) for hostname, site_id in self.matomo_sites_created],
level=2,
)
sites_ignored_summary = ''
if len(self.matomo_sites_ignored):
sites_ignored_summary = '\n' + self._indent_text(
self.matomo_sites_ignored, level=2,
)
summary = '''batch import statistics:
- imported: %(count_lines_recorded)d, downloads: %(count_lines_downloads)d
- ignored: %(total_lines_ignored)d
- HTTP errors: %(count_lines_skipped_http_errors)d, redirects: %(count_lines_skipped_http_redirects)d
- invalid: %(count_lines_invalid)d, filtered: %(count_lines_filtered)d
- no site match: %(count_lines_no_site)d, no hostname match: %(count_lines_hostname_skipped)d
- bots or search engines: %(count_lines_skipped_user_agent)d
- static: %(count_lines_static)d, downloads not matching any download extensions %(count_lines_skipped_downloads)d%(invalid_lines)s
- imported to: %(total_sites)d sites
- existing sites: %(total_sites_existing)d
- newly created sites: %(total_sites_created)d%(sites_created)s
- non-matching hostnames: %(total_sites_ignored)d%(sites_ignored)s
- time: %(total_time)ds, %(speed_recording)s reqs/second''' % {
'count_lines_recorded': self.count_lines_recorded.value,
'count_lines_downloads': self.count_lines_downloads.value,
'total_lines_ignored': sum([
self.count_lines_invalid.value,
self.count_lines_filtered.value,
self.count_lines_skipped_user_agent.value,
self.count_lines_skipped_http_errors.value,
self.count_lines_skipped_http_redirects.value,
self.count_lines_static.value,
self.count_lines_skipped_downloads.value,
self.count_lines_no_site.value,
self.count_lines_hostname_skipped.value,
]),
'count_lines_invalid': self.count_lines_invalid.value,
'count_lines_filtered': self.count_lines_filtered.value,
'count_lines_skipped_user_agent': self.count_lines_skipped_user_agent.value,
'count_lines_skipped_http_errors': self.count_lines_skipped_http_errors.value,
'count_lines_skipped_http_redirects': self.count_lines_skipped_http_redirects.value,
'count_lines_static': self.count_lines_static.value,
'count_lines_skipped_downloads': self.count_lines_skipped_downloads.value,
'count_lines_no_site': self.count_lines_no_site.value,
'count_lines_hostname_skipped': self.count_lines_hostname_skipped.value,
'total_sites': len(self.matomo_sites),
'total_sites_existing': len(self.matomo_sites - set(site_id for hostname, site_id in self.matomo_sites_created)),
'total_sites_created': len(self.matomo_sites_created),
'sites_created': sites_created_summary,
'total_sites_ignored': len(self.matomo_sites_ignored),
'sites_ignored': sites_ignored_summary,
'total_time': self.time_stop - self.time_start,
'speed_recording': self._round_value(self._compute_speed(
self.count_lines_recorded.value,
self.time_start, self.time_stop,
)),
'url': config.options.matomo_api_url,
'invalid_lines': invalid_lines_summary
}
for sline in summary.split('\n'):
logging.info(sline)
# monkey-patching fatal_error() so that we can handle ingestion errors gracefully
#
# first, define a replacement
......@@ -289,7 +364,7 @@ while True:
logging.info("ingesting logfiles...")
# we want stats
stats = import_logs.Statistics()
stats = Statistics()
import_logs.stats = stats
stats.set_time_start()
if config.options.show_progress:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment