split tsstats.py into package with multiple files

This commit is contained in:
Thor77 2016-05-08 21:32:37 +02:00
parent e779bbe2fc
commit a25a596d02
20 changed files with 333 additions and 322 deletions

View File

@ -1,3 +0,0 @@
[General]
logfile = tests/res/test.log
outputfile = tests/res/output.html

View File

@ -1,300 +0,0 @@
import argparse
import configparser
import datetime
import glob
import json
import logging
import re
from os import sep
from os.path import exists
from time import localtime, strftime
from jinja2 import Environment, FileSystemLoader
class Exceptions:
class ConfigNotFound(Exception):
pass
class InvalidConfig(Exception):
pass
class InvalidLog(Exception):
pass
exceptions = Exceptions
class Clients:
def __init__(self, ident_map={}):
self.clients_by_id = {}
self.clients_by_uid = {}
self.ident_map = ident_map
def is_id(self, id_or_uid):
try:
int(id_or_uid)
return True
except ValueError:
return False
def __add__(self, id_or_uid):
if self.is_id(id_or_uid):
if id_or_uid not in self.clients_by_id:
self.clients_by_id[id_or_uid] = Client(id_or_uid)
else:
if id_or_uid not in self.clients_by_uid:
self.clients_by_uid[id_or_uid] = Client(id_or_uid)
return self
def __getitem__(self, id_or_uid):
if id_or_uid in self.ident_map:
id_or_uid = self.ident_map[id_or_uid]
if self.is_id(id_or_uid):
if id_or_uid not in self.clients_by_id:
self += id_or_uid
return self.clients_by_id[id_or_uid]
else:
if id_or_uid not in self.clients_by_uid:
self += id_or_uid
return self.clients_by_uid[id_or_uid]
def __iter__(self):
for id_client in self.clients_by_id.values():
yield id_client
for uid_client in self.clients_by_uid.values():
yield uid_client
class Client:
def __init__(self, identifier):
# public
self.identifier = identifier
self.nick = None
self.connected = 0
self.onlinetime = 0
self.kicks = 0
self.pkicks = 0
self.bans = 0
self.pbans = 0
self.last_seen = 0
# private
self._last_connect = 0
def connect(self, timestamp):
'''
client connects at "timestamp"
'''
logging.debug('CONNECT {}'.format(str(self)))
self.connected += 1
self._last_connect = timestamp
def disconnect(self, timestamp):
'''
client disconnects at "timestamp"
'''
logging.debug('DISCONNECT {}'.format(str(self)))
if not self.connected:
logging.debug('^ disconnect before connect')
raise exceptions.InvalidLog('disconnect before connect!')
self.connected -= 1
session_time = timestamp - self._last_connect
self.onlinetime += session_time
self.last_seen = timestamp
def kick(self, target):
'''
client kicks "target" (Client-obj)
'''
logging.debug('KICK {} -> {}'.format(str(self), str(target)))
target.pkicks += 1
self.kicks += 1
def ban(self, target):
'''
client bans "target" (Client-obj)
'''
logging.debug('BAN {} -> {}'.format(str(self), str(target)))
target.pbans += 1
self.bans += 1
def __str__(self):
return '<{},{}>'.format(self.identifier, self.nick)
def __getitem__(self, item):
return {
'identifier': self.identifier,
'nick': self.nick,
'connected': self.connected,
'onlinetime': self.onlinetime,
'kicks': self.kicks,
'pkicks': self.pkicks,
'bans': self.bans,
'pbans': self.pbans,
}[item]
re_dis_connect = re.compile(r"'(.*)'\(id:(\d*)\)")
re_disconnect_invoker = re.compile(
r"invokername=(.*)\ invokeruid=(.*)\ reasonmsg"
)
path_split = __file__.split(sep)[:-1]
abspath = sep.join(path_split)
if len(path_split) > 0:
abspath += sep
def gen_abspath(filename):
return filename if filename.startswith(sep) else abspath + filename
def _get_sorted(stor, key):
clients = stor.values()
cl_data = [(client, client[key]) for client in clients if client[key] > 0]
return sorted(cl_data, key=lambda data: data[1], reverse=True)
def _format_seconds(seconds):
minutes, seconds = divmod(seconds, 60)
hours, minutes = divmod(minutes, 60)
hours = str(hours) + 'h ' if hours > 0 else ''
minutes = str(minutes) + 'm ' if minutes > 0 else ''
seconds = str(seconds) + 's' if seconds > 0 else ''
return hours + minutes + seconds
def parse_logs(log_path, ident_map={}, file_log=False):
clients = Clients(ident_map)
# setup logging
log = logging.getLogger()
log.setLevel(logging.DEBUG)
if file_log:
# file logger
file_handler = logging.FileHandler('debug.txt', 'w', 'UTF-8')
file_handler.setFormatter(logging.Formatter('%(message)s'))
file_handler.setLevel(logging.DEBUG)
log.addHandler(file_handler)
# stream logger (unused)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
log.addHandler(stream_handler)
# find all log-files and open them
file_paths = sorted([file_path for file_path in glob.glob(log_path)])
for file_path in file_paths:
log_file = open(file_path)
# process lines
logging.debug('Started parsing of {}'.format(log_file.name))
for line in log_file:
parts = line.split('|')
log_format = '%Y-%m-%d %H:%M:%S.%f'
stripped_time = datetime.datetime.strptime(parts[0], log_format)
logdatetime = int(stripped_time.timestamp())
data = '|'.join(parts[4:]).strip()
if data.startswith('client'):
nick, clid = re_dis_connect.findall(data)[0]
if data.startswith('client connected'):
client = clients[clid]
client.nick = nick
client.connect(logdatetime)
elif data.startswith('client disconnected'):
client = clients[clid]
client.nick = nick
client.disconnect(logdatetime)
if 'invokeruid' in data:
re_disconnect_data = re_disconnect_invoker.findall(
data)
invokernick, invokeruid = re_disconnect_data[0]
invoker = clients[invokeruid]
invoker.nick = invokernick
if 'bantime' in data:
invoker.ban(client)
else:
invoker.kick(client)
logging.debug('Finished parsing of {}'.format(log_file.name))
return clients
def render_template(clients, output, template_name='template.html',
title='TeamspeakStats', debug=False):
# prepare clients
clients_onlinetime_ = _get_sorted(clients.clients_by_id, 'onlinetime')
clients_onlinetime = [
(client, _format_seconds(onlinetime))
for client, onlinetime in clients_onlinetime_
]
clients_kicks = _get_sorted(clients.clients_by_uid, 'kicks')
clients_pkicks = _get_sorted(clients.clients_by_id, 'pkicks')
clients_bans = _get_sorted(clients.clients_by_uid, 'bans')
clients_pbans = _get_sorted(clients.clients_by_id, 'pbans')
objs = [('Onlinetime', clients_onlinetime), ('Kicks', clients_kicks),
('passive Kicks', clients_pkicks),
('Bans', clients_bans), ('passive Bans', clients_pbans)]
# render
template_loader = FileSystemLoader(abspath)
template_env = Environment(loader=template_loader)
def frmttime(timestamp):
return strftime('%x %X', localtime(int(timestamp)))
template_env.filters['frmttime'] = frmttime
template = template_env.get_template(template_name)
with open(output, 'w') as f:
f.write(template.render(title=title, objs=objs, debug=debug))
def parse_config(config_path):
config = configparser.ConfigParser()
config.read(config_path)
if 'General' not in config or not \
('logfile' in config['General'] and
'outputfile' in config['General']):
raise exceptions.InvalidConfig
general = config['General']
log_path = gen_abspath(general['logfile'])
output_path = gen_abspath(general['outputfile'])
return log_path, output_path
def main(config_path='config.ini', id_map_path='id_map.json',
debug=False, debugfile=False):
# check cmdline-args
config_path = gen_abspath(config_path)
id_map_path = gen_abspath(id_map_path)
if not exists(config_path):
raise exceptions.ConfigNotFound(config_path)
if exists(id_map_path):
# read id_map
id_map = json.load(open(id_map_path))
else:
id_map = {}
log_path, output_path = parse_config(config_path)
clients = parse_logs(log_path, ident_map=id_map, file_log=debugfile)
render_template(clients, output=output_path, debug=debug)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='A simple Teamspeak stats-generator - based on server-logs'
)
parser.add_argument(
'--config', type=str, help='path to config', default='config.ini'
)
parser.add_argument(
'--idmap', type=str, help='path to id_map', default='id_map.json'
)
parser.add_argument(
'--debug', help='debug mode', action='store_true'
)
parser.add_argument(
'--debugfile', help='write debug-log to file', action='store_true'
)
args = parser.parse_args()
main(args.config, args.idmap, args.debug, args.debugfile)

0
tsstats/__init__.py Normal file
View File

48
tsstats/__main__.py Normal file
View File

@ -0,0 +1,48 @@
import argparse
import json
from os.path import abspath, exists
from tsstats.config import parse_config
from tsstats.exceptions import ConfigNotFound
from tsstats.log import parse_logs
from tsstats.template import render_template
def main(config_path='config.ini', id_map_path='id_map.json',
debug=False, debugfile=False):
# check cmdline-args
config_path = abspath(config_path)
id_map_path = abspath(id_map_path)
if not exists(config_path):
raise ConfigNotFound(config_path)
if exists(id_map_path):
# read id_map
id_map = json.load(open(id_map_path))
else:
id_map = {}
log_path, output_path = parse_config(config_path)
clients = parse_logs(log_path, ident_map=id_map, file_log=debugfile)
render_template(clients, output=output_path, debug=debug)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='A simple Teamspeak stats-generator - based on server-logs'
)
parser.add_argument(
'--config', type=str, help='path to config', default='config.ini'
)
parser.add_argument(
'--idmap', type=str, help='path to id_map', default='id_map.json'
)
parser.add_argument(
'--debug', help='debug mode', action='store_true'
)
parser.add_argument(
'--debugfile', help='write debug-log to file', action='store_true'
)
args = parser.parse_args()
main(args.config, args.idmap, args.debug, args.debugfile)

114
tsstats/client.py Normal file
View File

@ -0,0 +1,114 @@
import logging
from tsstats.exceptions import InvalidLog
class Clients:
def __init__(self, ident_map={}):
self.clients_by_id = {}
self.clients_by_uid = {}
self.ident_map = ident_map
def is_id(self, id_or_uid):
try:
int(id_or_uid)
return True
except ValueError:
return False
def __add__(self, id_or_uid):
if self.is_id(id_or_uid):
if id_or_uid not in self.clients_by_id:
self.clients_by_id[id_or_uid] = Client(id_or_uid)
else:
if id_or_uid not in self.clients_by_uid:
self.clients_by_uid[id_or_uid] = Client(id_or_uid)
return self
def __getitem__(self, id_or_uid):
if id_or_uid in self.ident_map:
id_or_uid = self.ident_map[id_or_uid]
if self.is_id(id_or_uid):
if id_or_uid not in self.clients_by_id:
self += id_or_uid
return self.clients_by_id[id_or_uid]
else:
if id_or_uid not in self.clients_by_uid:
self += id_or_uid
return self.clients_by_uid[id_or_uid]
def __iter__(self):
for id_client in self.clients_by_id.values():
yield id_client
for uid_client in self.clients_by_uid.values():
yield uid_client
class Client:
def __init__(self, identifier):
# public
self.identifier = identifier
self.nick = None
self.connected = 0
self.onlinetime = 0
self.kicks = 0
self.pkicks = 0
self.bans = 0
self.pbans = 0
self.last_seen = 0
# private
self._last_connect = 0
def connect(self, timestamp):
'''
client connects at "timestamp"
'''
logging.debug('CONNECT {}'.format(str(self)))
self.connected += 1
self._last_connect = timestamp
def disconnect(self, timestamp):
'''
client disconnects at "timestamp"
'''
logging.debug('DISCONNECT {}'.format(str(self)))
if not self.connected:
logging.debug('^ disconnect before connect')
raise InvalidLog('disconnect before connect!')
self.connected -= 1
session_time = timestamp - self._last_connect
self.onlinetime += session_time
self.last_seen = timestamp
def kick(self, target):
'''
client kicks "target" (Client-obj)
'''
logging.debug('KICK {} -> {}'.format(str(self), str(target)))
target.pkicks += 1
self.kicks += 1
def ban(self, target):
'''
client bans "target" (Client-obj)
'''
logging.debug('BAN {} -> {}'.format(str(self), str(target)))
target.pbans += 1
self.bans += 1
def __str__(self):
return '<{},{}>'.format(self.identifier, self.nick)
def __getitem__(self, item):
return {
'identifier': self.identifier,
'nick': self.nick,
'connected': self.connected,
'onlinetime': self.onlinetime,
'kicks': self.kicks,
'pkicks': self.pkicks,
'bans': self.bans,
'pbans': self.pbans,
}[item]

22
tsstats/config.py Normal file
View File

@ -0,0 +1,22 @@
from os.path import abspath
from tsstats.exceptions import InvalidConfig
try:
from configparser import ConfigParser
except ImportError:
from ConfigParser import ConfigParser
def parse_config(config_path):
config = ConfigParser()
config.read(config_path)
if 'General' not in config or not \
('logfile' in config['General'] and
'outputfile' in config['General']):
raise InvalidConfig
general = config['General']
log_path = abspath(general['logfile'])
output_path = abspath(general['outputfile'])
return log_path, output_path

10
tsstats/exceptions.py Normal file
View File

@ -0,0 +1,10 @@
class InvalidConfig(Exception):
pass
class InvalidLog(Exception):
pass
class ConfigNotFound(Exception):
pass

64
tsstats/log.py Normal file
View File

@ -0,0 +1,64 @@
import logging
import re
from datetime import datetime
from glob import glob
from tsstats.client import Clients
re_dis_connect = re.compile(r"'(.*)'\(id:(\d*)\)")
re_disconnect_invoker = re.compile(
r'invokername=(.*)\ invokeruid=(.*)\ reasonmsg'
)
def parse_logs(log_path, ident_map={}, file_log=False):
clients = Clients(ident_map)
# setup logging
log = logging.getLogger()
log.setLevel(logging.DEBUG)
if file_log:
# file logger
file_handler = logging.FileHandler('debug.txt', 'w', 'UTF-8')
file_handler.setFormatter(logging.Formatter('%(message)s'))
file_handler.setLevel(logging.DEBUG)
log.addHandler(file_handler)
# stream logger (unused)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
log.addHandler(stream_handler)
# find all log-files and open them TODO: move this into main
file_paths = sorted([file_path for file_path in glob(log_path)])
for file_path in file_paths:
log_file = open(file_path)
# process lines
logging.debug('Started parsing of {}'.format(log_file.name))
for line in log_file:
parts = line.split('|')
log_format = '%Y-%m-%d %H:%M:%S.%f'
stripped_time = datetime.strptime(parts[0], log_format)
logdatetime = int(stripped_time.timestamp())
data = '|'.join(parts[4:]).strip()
if data.startswith('client'):
nick, clid = re_dis_connect.findall(data)[0]
if data.startswith('client connected'):
client = clients[clid]
client.nick = nick
client.connect(logdatetime)
elif data.startswith('client disconnected'):
client = clients[clid]
client.nick = nick
client.disconnect(logdatetime)
if 'invokeruid' in data:
re_disconnect_data = re_disconnect_invoker.findall(
data)
invokernick, invokeruid = re_disconnect_data[0]
invoker = clients[invokeruid]
invoker.nick = invokernick
if 'bantime' in data:
invoker.ban(client)
else:
invoker.kick(client)
logging.debug('Finished parsing of {}'.format(log_file.name))
return clients

35
tsstats/template.py Normal file
View File

@ -0,0 +1,35 @@
from os.path import abspath
from time import localtime, strftime
from jinja2 import Environment, FileSystemLoader
from tsstats.utils import seconds_to_text, sort_clients
def render_template(clients, output, template_name='tsstats/template.html',
title='TeamspeakStats', debug=False):
# prepare clients
clients_onlinetime_ = sort_clients(clients.clients_by_id, 'onlinetime')
clients_onlinetime = [
(client, seconds_to_text(onlinetime))
for client, onlinetime in clients_onlinetime_
]
clients_kicks = sort_clients(clients.clients_by_uid, 'kicks')
clients_pkicks = sort_clients(clients.clients_by_id, 'pkicks')
clients_bans = sort_clients(clients.clients_by_uid, 'bans')
clients_pbans = sort_clients(clients.clients_by_id, 'pbans')
objs = [('Onlinetime', clients_onlinetime), ('Kicks', clients_kicks),
('passive Kicks', clients_pkicks),
('Bans', clients_bans), ('passive Bans', clients_pbans)]
# render
template_loader = FileSystemLoader(abspath('.'))
template_env = Environment(loader=template_loader)
def fmttime(timestamp):
return strftime('%x %X', localtime(int(timestamp)))
template_env.filters['frmttime'] = fmttime
template = template_env.get_template(template_name)
with open(output, 'w') as f:
f.write(template.render(title=title, objs=objs, debug=debug))

View File

View File

@ -0,0 +1,3 @@
[General]
logfile = tsstats/tests/res/test.log
outputfile = tsstats/tests/res/output.html

View File

@ -1,12 +1,13 @@
import configparser
from os import remove
from os.path import exists
from os.path import abspath, exists
from nose.tools import raises, with_setup
from tsstats import exceptions, gen_abspath, parse_config
from tsstats import exceptions
from tsstats.config import parse_config
configpath = gen_abspath('tests/res/test.cfg')
configpath = abspath('tsstats/tests/res/test.cfg')
def create_config(values, key='General'):
@ -25,7 +26,7 @@ def clean_config():
@raises(exceptions.InvalidConfig)
def test_invalid_config():
create_config({
'loggfile': 'tests/res/test.log',
'loggfile': 'tsstats/tests/res/test.log',
'outputfile': ''
})
_, _, _, _ = parse_config(configpath)
@ -34,10 +35,10 @@ def test_invalid_config():
@with_setup(clean_config, clean_config)
def test_config():
create_config({
'logfile': 'tests/res/test.log',
'logfile': 'tsstats/tests/res/test.log',
'outputfile': 'output.html',
'debug': 'true'
})
log_path, output_path = parse_config(configpath)
assert log_path == gen_abspath('tests/res/test.log')
assert output_path == gen_abspath('output.html')
assert log_path == abspath('tsstats/tests/res/test.log')
assert output_path == abspath('output.html')

View File

@ -2,13 +2,15 @@ from os import remove
from nose.tools import raises
from tsstats import exceptions, main, parse_logs
from tsstats import exceptions
from tsstats.__main__ import main
from tsstats.log import parse_logs
clients = parse_logs('tests/res/test.log')
clients = parse_logs('tsstats/tests/res/test.log')
def test_main():
main(config_path='tests/res/config.ini')
main(config_path='tsstats/tests/res/config.ini')
@raises(exceptions.ConfigNotFound)
@ -17,8 +19,8 @@ def test_main_config_not_found():
def test_main_idmap_load():
main(config_path='tests/res/config.ini',
id_map_path='tests/res/id_map.json')
main(config_path='tsstats/tests/res/config.ini',
id_map_path='tsstats/tests/res/id_map.json')
def test_length():
@ -58,14 +60,14 @@ def test_client_repr():
def test_debug_log():
clients = parse_logs('tests/res/test.log', file_log=True)
clients = parse_logs('tsstats/tests/res/test.log', file_log=True)
open('debug.txt')
remove('debug.txt')
@raises(exceptions.InvalidLog)
def test_parse_broken():
clients = parse_logs('tests/res/test.log.broken')
clients = parse_logs('tsstats/tests/res/test.log.broken')
def test_iter_clients():

View File

@ -1,4 +1,4 @@
from tsstats import Clients
from tsstats.client import Clients
ident_map = {
'1': '2',

View File

@ -2,10 +2,12 @@ from os import remove
from bs4 import BeautifulSoup
from tsstats import _format_seconds, parse_logs, render_template
from tsstats.utils import seconds_to_text
from tsstats.log import parse_logs
from tsstats.template import render_template
output_path = 'tests/res/output.html'
clients = parse_logs('tests/res/test.log')
output_path = 'tsstats/tests/res/output.html'
clients = parse_logs('tsstats/tests/res/test.log')
class TestTemplate:
@ -26,5 +28,5 @@ class TestTemplate:
render_template(clients, output_path)
soup = BeautifulSoup(open(output_path), 'html.parser')
# check onlinetime-data
assert _format_seconds(clients['1'].onlinetime) == \
assert seconds_to_text(clients['1'].onlinetime) == \
soup.find('span', class_='badge').text

13
tsstats/utils.py Normal file
View File

@ -0,0 +1,13 @@
def sort_clients(stor, key):
clients = stor.values()
cl_data = [(client, client[key]) for client in clients if client[key] > 0]
return sorted(cl_data, key=lambda data: data[1], reverse=True)
def seconds_to_text(seconds):
minutes, seconds = divmod(seconds, 60)
hours, minutes = divmod(minutes, 60)
hours = str(hours) + 'h ' if hours > 0 else ''
minutes = str(minutes) + 'm ' if minutes > 0 else ''
seconds = str(seconds) + 's' if seconds > 0 else ''
return hours + minutes + seconds