diff --git a/tsstats/__main__.py b/tsstats/__main__.py
index 5e7e621..e469b92 100644
--- a/tsstats/__main__.py
+++ b/tsstats/__main__.py
@@ -69,8 +69,13 @@ def main(config=None, idmap=None, log=None,
if not log or not output:
raise InvalidConfiguration('log or output missing')
- clients = parse_logs(log, ident_map=identmap, online_dc=noonlinedc)
- render_template(clients, output=abspath(output))
+ sid_clients = parse_logs(log, ident_map=identmap, online_dc=noonlinedc)
+ for sid, clients in sid_clients.items():
+ if sid:
+ ext = '.{}'.format(sid)
+ else:
+ ext = ''
+ render_template(clients, output=abspath(output + ext))
if __name__ == '__main__':
diff --git a/tsstats/log.py b/tsstats/log.py
index 08570f6..5b3bf43 100644
--- a/tsstats/log.py
+++ b/tsstats/log.py
@@ -2,11 +2,15 @@
import logging
import re
+from collections import namedtuple
from datetime import datetime
from glob import glob
+from os.path import basename
from tsstats.client import Client, Clients
+re_log_filename = re.compile(r'ts3server_(?P<date>\d{4}-\d\d-\d\d)'
+ '__(?P<time>\d\d_\d\d_\d\d.\d+)_(?P<sid>\d).log')
re_log_entry = re.compile('(?P<timestamp>\d{4}-\d\d-\d\d\ \d\d:\d\d:\d\d.\d+)'
'\|\ *(?P<level>\w+)\ *\|\ *(?P<component>\w+)\ *'
'\|\ *(?P<sid>\d+)\ *\|\ *(?P<message>.*)')
@@ -17,32 +21,83 @@ re_disconnect_invoker = re.compile(
log_timestamp_format = '%Y-%m-%d %H:%M:%S.%f'
+TimedLog = namedtuple('TimedLog', ['path', 'timestamp'])
+
logger = logging.getLogger('tsstats')
def parse_logs(log_glob, ident_map=None, *args, **kwargs):
'''
- parse logs specified by globbing pattern `log_glob`
+ parse logs from `log_glob`
- :param log_glob: path to log-files (supports globbing)
- :param ident_map: :doc:`identmap`
+ :param log_glob: path to server-logs (supports globbing)
+ :param ident_map: identmap used for Client-initializations
:type log_glob: str
:type ident_map: dict
- :return: parsed clients
- :rtype: tsstats.client.Clients
+ :return: clients bundled by virtual-server
+ :rtype: dict
'''
- clients = Clients(ident_map)
- for log_file in sorted(log_file for log_file in glob(log_glob)):
- clients = parse_log(log_file, ident_map, clients, *args, **kwargs)
- return clients
+ vserver_clients = {}
+ for virtualserver_id, logs in\
+ _bundle_logs(log_file for log_file in glob(log_glob)).items():
+ clients = Clients(ident_map)
+ for log in logs:
+ _parse_details(log.path, clients=clients, *args, **kwargs)
+ if len(clients) >= 1:
+ vserver_clients[virtualserver_id] = clients
+ return vserver_clients
-def parse_log(log_path, ident_map=None, clients=None, online_dc=True):
+def _bundle_logs(logs):
'''
- parse log-file at `log_path`
+ bundle `logs` by virtualserver-id
+ and sort by timestamp from filename (if exists)
+
+ :param logs: list of paths to logfiles
+
+ :type logs: list
+
+ :return: `logs` bundled by virtualserver-id and sorted by timestamp
+ :rtype: dict{str: [TimedLog]}
+ '''
+ vserver_logfiles = {} # sid: [/path/to/log1, ..., /path/to/logn]
+ for log in logs:
+ # try to get date and sid from filename
+ match = re_log_filename.match(basename(log))
+ if match:
+ match = match.groupdict()
+ timestamp = datetime.strptime('{0} {1}'.format(
+ match['date'], match['time'].replace('_', ':')),
+ log_timestamp_format)
+ tl = TimedLog(log, timestamp)
+ sid = match['sid']
+ if sid in vserver_logfiles:
+ # if already exists, keep list sorted by timestamp
+ vserver_logfiles[sid].append(tl)
+ vserver_logfiles[sid] =\
+ sorted(vserver_logfiles[sid],
+ key=lambda tl: tl.timestamp)
+ else:
+ # if not exists, just create a list
+ vserver_logfiles[match['sid']] = [tl]
+ else:
+ # fallback to plain sorting
+ vserver_logfiles.setdefault('', [])\
+ .append(TimedLog(log, None))
+ vserver_logfiles[''] =\
+ sorted(vserver_logfiles[''],
+ key=lambda tl: tl.path)
+ return vserver_logfiles
+
+
+def _parse_details(log_path, ident_map=None, clients=None, online_dc=True):
+ '''
+ extract details from log-files
+
+ detailed parsing is done here: onlinetime, kicks, pkicks, bans, pbans
:param log_path: path to log-file
:param ident_map: :doc:`identmap`
@@ -57,7 +112,7 @@ def parse_log(log_path, ident_map=None, clients=None, online_dc=True):
:return: parsed clients
:rtype: tsstats.client.Clients
'''
- if not clients:
+ if clients is None:
clients = Clients(ident_map)
log_file = open(log_path)
# process lines
diff --git a/tsstats/tests/test_log.py b/tsstats/tests/test_log.py
index cd0a62f..1dc1689 100644
--- a/tsstats/tests/test_log.py
+++ b/tsstats/tests/test_log.py
@@ -1,17 +1,17 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
from time import sleep
import pytest
from tsstats.exceptions import InvalidLog
-from tsstats.log import parse_log, parse_logs
+from tsstats.log import TimedLog, _bundle_logs, _parse_details, parse_logs
testlog_path = 'tsstats/tests/res/test.log'
@pytest.fixture
def clients():
- return parse_log(testlog_path, online_dc=False)
+ return _parse_details(testlog_path, online_dc=False)
def test_log_client_count(clients):
@@ -39,20 +39,46 @@ def test_log_pbans(clients):
assert clients['2'].pbans == 1
+@pytest.mark.parametrize("logs,bundled", [
+ (
+ ['l1.log', 'l2.log'],
+ {'': [TimedLog('l1.log', None), TimedLog('l2.log', None)]}
+ ),
+ (
+ [
+ 'ts3server_2016-06-06__14_22_09.527229_1.log',
+ 'ts3server_2017-07-07__15_23_10.638340_1.log'
+ ],
+ {
+ '1': [
+ TimedLog('ts3server_2016-06-06__14_22_09.527229_1.log',
+ datetime(year=2016, month=6, day=6, hour=14,
+ minute=22, second=9, microsecond=527229)),
+ TimedLog('ts3server_2017-07-07__15_23_10.638340_1.log',
+ datetime(year=2017, month=7, day=7, hour=15,
+ minute=23, second=10, microsecond=638340))
+ ]
+ }
+ )
+])
+def test_log_bundle(logs, bundled):
+ assert _bundle_logs(logs) == bundled
+
+
def test_log_invalid():
with pytest.raises(InvalidLog):
- parse_log('tsstats/tests/res/test.log.broken')
-
-
-def test_log_multiple():
- assert len(parse_log(testlog_path, online_dc=False)) == \
- len(parse_logs(testlog_path, online_dc=False))
+ _parse_details('tsstats/tests/res/test.log.broken')
@pytest.mark.slowtest
def test_log_client_online():
- clients = parse_log(testlog_path)
+ clients = _parse_details(testlog_path)
old_onlinetime = int(clients['1'].onlinetime.total_seconds())
sleep(2)
- clients = parse_log(testlog_path)
+ clients = _parse_details(testlog_path)
assert int(clients['1'].onlinetime.total_seconds()) == old_onlinetime + 2
+
+
+def test_parse_logs():
+ assert len(_parse_details(testlog_path)) ==\
+ len(parse_logs(testlog_path)[''])
diff --git a/tsstats/tests/test_template.py b/tsstats/tests/test_template.py
index 9652cd5..8e90b45 100644
--- a/tsstats/tests/test_template.py
+++ b/tsstats/tests/test_template.py
@@ -5,12 +5,12 @@ from os import remove
import pytest
from bs4 import BeautifulSoup
-from tsstats.log import parse_log
+from tsstats.log import _parse_details
from tsstats.template import render_template
from tsstats.utils import seconds_to_text
output_path = 'tsstats/tests/res/output.html'
-clients = parse_log('tsstats/tests/res/test.log')
+clients = _parse_details('tsstats/tests/res/test.log')
logger = logging.getLogger('tsstats')