Merge branch 'multi_vserver'

tsstats/__main__.py

@@ -69,8 +69,13 @@ def main(config=None, idmap=None, log=None,
     if not log or not output:
         raise InvalidConfiguration('log or output missing')
 
-    clients = parse_logs(log, ident_map=identmap, online_dc=noonlinedc)
+    sid_clients = parse_logs(log, ident_map=identmap, online_dc=noonlinedc)
-    render_template(clients, output=abspath(output))
+    for sid, clients in sid_clients.items():
+        if sid:
+            ext = '.{}'.format(sid)
+        else:
+            ext = ''
+        render_template(clients, output=abspath(output + ext))
 
 
 if __name__ == '__main__':

tsstats/log.py

@@ -2,11 +2,15 @@
 
 import logging
 import re
+from collections import namedtuple
 from datetime import datetime
 from glob import glob
+from os.path import basename
 
 from tsstats.client import Client, Clients
 
+re_log_filename = re.compile(r'ts3server_(?P<date>\d{4}-\d\d-\d\d)'
+                             '__(?P<time>\d\d_\d\d_\d\d.\d+)_(?P<sid>\d).log')
 re_log_entry = re.compile('(?P<timestamp>\d{4}-\d\d-\d\d\ \d\d:\d\d:\d\d.\d+)'
                           '\|\ *(?P<level>\w+)\ *\|\ *(?P<component>\w+)\ *'
                           '\|\ *(?P<sid>\d+)\ *\|\ *(?P<message>.*)')
@@ -17,32 +21,83 @@ re_disconnect_invoker = re.compile(
 
 log_timestamp_format = '%Y-%m-%d %H:%M:%S.%f'
 
+TimedLog = namedtuple('TimedLog', ['path', 'timestamp'])
+
 
 logger = logging.getLogger('tsstats')
 
 
 def parse_logs(log_glob, ident_map=None, *args, **kwargs):
     '''
-    parse logs specified by globbing pattern `log_glob`
+    parse logs from `log_glob`
 
-    :param log_glob: path to log-files (supports globbing)
+    :param log_glob: path to server-logs (supports globbing)
-    :param ident_map: :doc:`identmap`
+    :param ident_map: identmap used for Client-initializations
 
     :type log_glob: str
     :type ident_map: dict
 
-    :return: parsed clients
+    :return: clients bundled by virtual-server
-    :rtype: tsstats.client.Clients
+    :rtype: dict
     '''
+    vserver_clients = {}
+    for virtualserver_id, logs in\
+            _bundle_logs(log_file for log_file in glob(log_glob)).items():
         clients = Clients(ident_map)
-    for log_file in sorted(log_file for log_file in glob(log_glob)):
+        for log in logs:
-        clients = parse_log(log_file, ident_map, clients, *args, **kwargs)
+            _parse_details(log.path, clients=clients, *args, **kwargs)
-    return clients
+        if len(clients) >= 1:
+            vserver_clients[virtualserver_id] = clients
+    return vserver_clients
 
 
-def parse_log(log_path, ident_map=None, clients=None, online_dc=True):
+def _bundle_logs(logs):
     '''
-    parse log-file at `log_path`
+    bundle `logs` by virtualserver-id
+    and sort by timestamp from filename (if exists)
+
+    :param logs: list of paths to logfiles
+
+    :type logs: list
+
+    :return: `logs` bundled by virtualserver-id and sorted by timestamp
+    :rtype: dict{str: [TimedLog]}
+    '''
+    vserver_logfiles = {}  # sid: [/path/to/log1, ..., /path/to/logn]
+    for log in logs:
+        # try to get date and sid from filename
+        match = re_log_filename.match(basename(log))
+        if match:
+            match = match.groupdict()
+            timestamp = datetime.strptime('{0} {1}'.format(
+                match['date'], match['time'].replace('_', ':')),
+                log_timestamp_format)
+            tl = TimedLog(log, timestamp)
+            sid = match['sid']
+            if sid in vserver_logfiles:
+                # if already exists, keep list sorted by timestamp
+                vserver_logfiles[sid].append(tl)
+                vserver_logfiles[sid] =\
+                    sorted(vserver_logfiles[sid],
+                           key=lambda tl: tl.timestamp)
+            else:
+                # if not exists, just create a list
+                vserver_logfiles[match['sid']] = [tl]
+        else:
+            # fallback to plain sorting
+            vserver_logfiles.setdefault('', [])\
+                .append(TimedLog(log, None))
+            vserver_logfiles[''] =\
+                sorted(vserver_logfiles[''],
+                       key=lambda tl: tl.path)
+    return vserver_logfiles
+
+
+def _parse_details(log_path, ident_map=None, clients=None, online_dc=True):
+    '''
+    extract details from log-files
+
+    detailed parsing is done here: onlinetime, kicks, pkicks, bans, pbans
 
     :param log_path: path to log-file
     :param ident_map: :doc:`identmap`
@@ -57,7 +112,7 @@ def parse_log(log_path, ident_map=None, clients=None, online_dc=True):
     :return: parsed clients
     :rtype: tsstats.client.Clients
     '''
-    if not clients:
+    if clients is None:
         clients = Clients(ident_map)
     log_file = open(log_path)
     # process lines

tsstats/tests/test_log.py

@@ -1,17 +1,17 @@
-from datetime import timedelta
+from datetime import datetime, timedelta
 from time import sleep
 
 import pytest
 
 from tsstats.exceptions import InvalidLog
-from tsstats.log import parse_log, parse_logs
+from tsstats.log import TimedLog, _bundle_logs, _parse_details, parse_logs
 
 testlog_path = 'tsstats/tests/res/test.log'
 
 
 @pytest.fixture
 def clients():
-    return parse_log(testlog_path, online_dc=False)
+    return _parse_details(testlog_path, online_dc=False)
 
 
 def test_log_client_count(clients):
@@ -39,20 +39,46 @@ def test_log_pbans(clients):
     assert clients['2'].pbans == 1
 
 
+@pytest.mark.parametrize("logs,bundled", [
+    (
+        ['l1.log', 'l2.log'],
+        {'': [TimedLog('l1.log', None), TimedLog('l2.log', None)]}
+    ),
+    (
+        [
+            'ts3server_2016-06-06__14_22_09.527229_1.log',
+            'ts3server_2017-07-07__15_23_10.638340_1.log'
+        ],
+        {
+            '1': [
+                TimedLog('ts3server_2016-06-06__14_22_09.527229_1.log',
+                         datetime(year=2016, month=6, day=6, hour=14,
+                                  minute=22, second=9, microsecond=527229)),
+                TimedLog('ts3server_2017-07-07__15_23_10.638340_1.log',
+                         datetime(year=2017, month=7, day=7, hour=15,
+                                  minute=23, second=10, microsecond=638340))
+            ]
+        }
+    )
+])
+def test_log_bundle(logs, bundled):
+    assert _bundle_logs(logs) == bundled
+
+
 def test_log_invalid():
     with pytest.raises(InvalidLog):
-        parse_log('tsstats/tests/res/test.log.broken')
+        _parse_details('tsstats/tests/res/test.log.broken')
-
-
-def test_log_multiple():
-    assert len(parse_log(testlog_path, online_dc=False)) == \
-        len(parse_logs(testlog_path, online_dc=False))
 
 
 @pytest.mark.slowtest
 def test_log_client_online():
-    clients = parse_log(testlog_path)
+    clients = _parse_details(testlog_path)
     old_onlinetime = int(clients['1'].onlinetime.total_seconds())
     sleep(2)
-    clients = parse_log(testlog_path)
+    clients = _parse_details(testlog_path)
     assert int(clients['1'].onlinetime.total_seconds()) == old_onlinetime + 2
+
+
+def test_parse_logs():
+    assert len(_parse_details(testlog_path)) ==\
+        len(parse_logs(testlog_path)[''])

tsstats/tests/test_template.py

@@ -5,12 +5,12 @@ from os import remove
 import pytest
 from bs4 import BeautifulSoup
 
-from tsstats.log import parse_log
+from tsstats.log import _parse_details
 from tsstats.template import render_template
 from tsstats.utils import seconds_to_text
 
 output_path = 'tsstats/tests/res/output.html'
-clients = parse_log('tsstats/tests/res/test.log')
+clients = _parse_details('tsstats/tests/res/test.log')
 
 logger = logging.getLogger('tsstats')