feat(logs): rewrite Logs cog

This commit is contained in:
Romain J 2020-10-22 00:00:48 +02:00
parent bdd77d1841
commit 71335de878
17 changed files with 529 additions and 70 deletions

View file

@ -38,6 +38,7 @@
<w>tuxbot's</w>
<w>tuxvenv</w>
<w>venv</w>
<w>webhook</w>
<w>webhooks</w>
<w>écrite</w>
</words>

View file

@ -4,6 +4,7 @@ good-names=
f, # (file) as f
k, # for k, v in
v, # for k, v in
dt, # datetime
[MASTER]
disable=
@ -11,5 +12,6 @@ disable=
C0115, # missing-class-docstring
C0116, # missing-function-docstring
W0703, # broad-except
R0801, # duplicate-code
R0902, # too-many-instance-attributes
R0903, # too-few-public-methods

View file

@ -1,5 +1,3 @@
from typing import NoReturn
from rich.console import Console
from rich.traceback import install
from tuxbot import ExitCodes
@ -8,7 +6,7 @@ console = Console()
install(console=console)
def main() -> NoReturn:
def main() -> None:
try:
from .__run__ import run # pylint: disable=import-outside-toplevel

View file

@ -6,7 +6,6 @@ import sys
import os
import tracemalloc
from argparse import Namespace
from typing import NoReturn
from datetime import datetime
import discord
@ -35,7 +34,7 @@ tracemalloc.start()
BORDER_STYLE = "not dim"
def list_instances() -> NoReturn:
def list_instances() -> None:
"""List all available instances"""
app_config = config.ConfigFile(
data_manager.config_dir / "config.yaml", config.AppConfig
@ -70,7 +69,7 @@ def list_instances() -> NoReturn:
sys.exit(os.EX_OK)
def debug_info() -> NoReturn:
def debug_info() -> None:
"""Show debug info relatives to the bot"""
python_version = sys.version.replace("\n", "")
pip_version = pip.__version__
@ -172,7 +171,7 @@ def parse_cli_flags(args: list) -> Namespace:
return args
async def shutdown_handler(tux: Tux, signal_type, exit_code=None) -> NoReturn:
async def shutdown_handler(tux: Tux, signal_type, exit_code=None) -> None:
"""Handler when the bot shutdown
It cancels all running task.
@ -247,7 +246,7 @@ async def run_bot(tux: Tux, cli_flags: Namespace) -> None:
return None
def run() -> NoReturn:
def run() -> None:
"""Main function"""
tux = None
cli_flags = parse_cli_flags(sys.argv[1:])

View file

@ -1,36 +1,8 @@
from structured_config import Structure, StrField
from structured_config import Structure
class AdminConfig(Structure):
dm: str = StrField("")
mentions: str = StrField("")
guilds: str = StrField("")
errors: str = StrField("")
gateway: str = StrField("")
pass
extra = {
"dm": {
"type": str,
"description": "URL of the webhook used for send DMs "
"received and sent by the bot",
},
"mentions": {
"type": str,
"description": "URL of the webhook used for send Mentions "
"received by the bot",
},
"guilds": {
"type": str,
"description": "URL of the webhook used for send guilds where the "
"bot is added or removed",
},
"errors": {
"type": str,
"description": "URL of the webhook used for send errors in the bot",
},
"gateway": {
"type": str,
"description": "URL of the webhook used for send gateway information",
},
}
extra = {}

View file

@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: Tuxbot-bot\n"
"Report-Msgid-Bugs-To: rick@gnous.eu\n"
"POT-Creation-Date: 2020-06-11 19:07+0200\n"
"POT-Creation-Date: 2020-10-21 01:13+0200\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@ -17,11 +17,11 @@ msgstr ""
"Content-Type: text/plain; charset=CHARSET\n"
"Content-Transfer-Encoding: 8bit\n"
#: tuxbot/cogs/admin/admin.py:33
#: tuxbot/cogs/admin/admin.py:47
#, python-brace-format
msgid "Locale changed to {lang} successfully"
msgstr ""
#: tuxbot/cogs/admin/admin.py:43
#: tuxbot/cogs/admin/admin.py:62
msgid "List of available locales: "
msgstr ""

View file

@ -0,0 +1,27 @@
import logging
from collections import namedtuple
from discord.ext import commands
from .logs import Logs, on_error, GatewayHandler
from .config import LogsConfig
from ...core.bot import Tux
VersionInfo = namedtuple("VersionInfo", "major minor micro release_level")
version_info = VersionInfo(major=1, minor=0, micro=0, release_level="alpha")
__version__ = "v{}.{}.{}-{}".format(
version_info.major,
version_info.minor,
version_info.micro,
version_info.release_level,
).replace("\n", "")
def setup(bot: Tux):
cog = Logs(bot)
bot.add_cog(cog)
handler = GatewayHandler(cog)
logging.getLogger().addHandler(handler)
commands.AutoShardedBot.on_error = on_error

View file

@ -0,0 +1,36 @@
from structured_config import Structure, StrField
class LogsConfig(Structure):
dm: str = StrField("")
mentions: str = StrField("")
guilds: str = StrField("")
errors: str = StrField("")
gateway: str = StrField("")
extra = {
"dm": {
"type": str,
"description": "URL of the webhook used for send DMs "
"received and sent by the bot",
},
"mentions": {
"type": str,
"description": "URL of the webhook used for send Mentions "
"received by the bot",
},
"guilds": {
"type": str,
"description": "URL of the webhook used for send guilds where the "
"bot is added or removed",
},
"errors": {
"type": str,
"description": "URL of the webhook used for send errors in the bot",
},
"gateway": {
"type": str,
"description": "URL of the webhook used for send gateway information",
},
}

View file

@ -0,0 +1,18 @@
# English translations for Tuxbot-bot package.
# Copyright (C) 2020 THE Tuxbot-bot'S COPYRIGHT HOLDER
# This file is distributed under the same license as the Tuxbot-bot package.
# Automatically generated, 2020.
#
msgid ""
msgstr ""
"Project-Id-Version: Tuxbot-bot\n"
"Report-Msgid-Bugs-To: rick@gnous.eu\n"
"POT-Creation-Date: 2020-10-21 01:15+0200\n"
"PO-Revision-Date: 2020-10-21 01:15+0200\n"
"Last-Translator: Automatically generated\n"
"Language-Team: none\n"
"Language: en_US\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"

View file

@ -0,0 +1,19 @@
# French translations for Tuxbot-bot package
# Traductions françaises du paquet Tuxbot-bot.
# Copyright (C) 2020 THE Tuxbot-bot'S COPYRIGHT HOLDER
# This file is distributed under the same license as the Tuxbot-bot package.
# Automatically generated, 2020.
#
msgid ""
msgstr ""
"Project-Id-Version: Tuxbot-bot\n"
"Report-Msgid-Bugs-To: rick@gnous.eu\n"
"POT-Creation-Date: 2020-10-21 01:15+0200\n"
"PO-Revision-Date: 2020-10-21 01:15+0200\n"
"Last-Translator: Automatically generated\n"
"Language-Team: none\n"
"Language: en_US\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"

View file

@ -0,0 +1,18 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the Tuxbot-bot package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: Tuxbot-bot\n"
"Report-Msgid-Bugs-To: rick@gnous.eu\n"
"POT-Creation-Date: 2020-10-21 01:15+0200\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
"Language: \n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=CHARSET\n"
"Content-Transfer-Encoding: 8bit\n"

332
tuxbot/cogs/logs/logs.py Normal file
View file

@ -0,0 +1,332 @@
import asyncio
import datetime
import json
import logging
import textwrap
import traceback
from collections import defaultdict
from logging import LogRecord
import discord
import humanize
import psutil
from discord.ext import commands, tasks
from structured_config import ConfigFile
from tuxbot.core.bot import Tux
from tuxbot.core.i18n import (
Translator,
)
from tuxbot.core.utils.functions.extra import (
command_extra,
ContextPlus,
)
from .config import LogsConfig
from ...core.data_manager import cogs_data_path
log = logging.getLogger("tuxbot.cogs.logs")
_ = Translator("Logs", __file__)
class GatewayHandler(logging.Handler):
def __init__(self, cog):
self.cog = cog
super().__init__(logging.INFO)
def filter(self, record: LogRecord):
return (
record.name == "discord.gateway"
or "Shard ID" in record.msg
or "Websocket closed " in record.msg
)
def emit(self, record: LogRecord):
self.cog.add_record(record)
class Logs(commands.Cog, name="Logs"):
def __init__(self, bot: Tux):
self.bot = bot
self.process = psutil.Process()
self._batch_lock = asyncio.Lock(loop=bot.loop)
self._data_batch = []
self._gateway_queue = asyncio.Queue(loop=bot.loop)
self.gateway_worker.start() # pylint: disable=no-member
self.config: LogsConfig = ConfigFile(
str(
cogs_data_path(self.bot.instance_name, "logs") / "config.yaml"
),
LogsConfig,
).config
self._resumes = []
self._identifies = defaultdict(list)
def _clear_gateway_data(self):
one_week_ago = datetime.datetime.utcnow() - datetime.timedelta(days=7)
to_remove = [
index
for index, dt in enumerate(self._resumes)
if dt < one_week_ago
]
for index in reversed(to_remove):
del self._resumes[index]
for _, dates in self._identifies.items():
to_remove = [
index for index, dt in enumerate(dates) if dt < one_week_ago
]
for index in reversed(to_remove):
del dates[index]
@tasks.loop(seconds=0.0)
async def gateway_worker(self):
record = await self._gateway_queue.get()
await self.notify_gateway_status(record)
async def register_command(self, ctx: ContextPlus):
if ctx.command is None:
return
command = ctx.command.qualified_name
self.bot.stats["commands"][command] += 1
message = ctx.message
if ctx.guild is None:
destination = "Private Message"
guild_id = None
else:
destination = f"#{message.channel} ({message.guild})"
guild_id = ctx.guild.id
log.info(
"%s: %s in %s > %s",
message.created_at,
message.author,
destination,
message.content,
)
async with self._batch_lock:
self._data_batch.append(
{
"guild": guild_id,
"channel": ctx.channel.id,
"author": ctx.author.id,
"used": message.created_at.isoformat(),
"prefix": ctx.prefix,
"command": command,
"failed": ctx.command_failed,
}
)
@commands.Cog.listener()
async def on_command_completion(self, ctx: ContextPlus):
await self.register_command(ctx)
@commands.Cog.listener()
async def on_socket_response(self, msg):
self.bot.stats["socket"][msg.get("t")] += 1
def webhook(self, log_type):
webhook = discord.Webhook.from_url(
getattr(self.config, log_type),
adapter=discord.AsyncWebhookAdapter(self.bot.session),
)
return webhook
async def log_error(self, *, ctx: ContextPlus = None, extra=None):
e = discord.Embed(title="Error", colour=0xDD5F53)
e.description = f"```py\n{traceback.format_exc()}\n```"
e.add_field(name="Extra", value=extra, inline=False)
e.timestamp = datetime.datetime.utcnow()
if ctx is not None:
fmt = "{0} (ID: {0.id})"
author = fmt.format(ctx.author)
channel = fmt.format(ctx.channel)
guild = "None" if ctx.guild is None else fmt.format(ctx.guild)
e.add_field(name="Author", value=author)
e.add_field(name="Channel", value=channel)
e.add_field(name="Guild", value=guild)
await self.webhook("errors").send(embed=e)
async def send_guild_stats(self, e, guild):
e.add_field(name="Name", value=guild.name)
e.add_field(name="ID", value=guild.id)
e.add_field(name="Shard ID", value=guild.shard_id or "N/A")
e.add_field(
name="Owner", value=f"{guild.owner} (ID: {guild.owner.id})"
)
bots = sum(member.bot for member in guild.members)
total = guild.member_count
online = sum(
member.status is discord.Status.online for member in guild.members
)
e.add_field(name="Members", value=str(total))
e.add_field(name="Bots", value=f"{bots} ({bots / total:.2%})")
e.add_field(name="Online", value=f"{online} ({online / total:.2%})")
if guild.icon:
e.set_thumbnail(url=guild.icon_url)
if guild.me:
e.timestamp = guild.me.joined_at
await self.webhook("guilds").send(embed=e)
@commands.Cog.listener()
async def on_guild_join(self, guild: discord.guild):
e = discord.Embed(colour=0x53DDA4, title="New Guild") # green colour
await self.send_guild_stats(e, guild)
@commands.Cog.listener()
async def on_guild_remove(self, guild: discord.guild):
e = discord.Embed(colour=0xDD5F53, title="Left Guild") # red colour
await self.send_guild_stats(e, guild)
@commands.Cog.listener()
async def on_message(self, message: discord.message):
if message.guild is None:
e = discord.Embed(colour=0x0A97F5, title="New DM") # blue colour
e.set_author(
name=message.author,
icon_url=message.author.avatar_url_as(format="png"),
)
e.description = message.content
if len(message.attachments) > 0:
e.set_image(url=message.attachments[0].url)
e.set_footer(text=f"User ID: {message.author.id}")
await self.webhook("dm").send(embed=e)
@commands.Cog.listener()
async def on_command_error(self, ctx: ContextPlus, error):
await self.register_command(ctx)
if not isinstance(
error, (commands.CommandInvokeError, commands.ConversionError)
):
return
error = error.original
if isinstance(error, (discord.Forbidden, discord.NotFound)):
return
e = discord.Embed(title="Command Error", colour=0xCC3366)
e.add_field(name="Name", value=ctx.command.qualified_name)
e.add_field(name="Author", value=f"{ctx.author} (ID: {ctx.author.id})")
fmt = f"Channel: {ctx.channel} (ID: {ctx.channel.id})"
if ctx.guild:
fmt = f"{fmt}\nGuild: {ctx.guild} (ID: {ctx.guild.id})"
e.add_field(name="Location", value=fmt, inline=False)
e.add_field(
name="Content",
value=textwrap.shorten(ctx.message.content, width=512),
)
exc = "".join(
traceback.format_exception(
type(error), error, error.__traceback__, chain=False
)
)
e.description = f"```py\n{exc}\n```"
e.timestamp = datetime.datetime.utcnow()
await self.webhook("errors").send(embed=e)
@commands.Cog.listener()
async def on_socket_raw_send(self, data):
if '"op":2' not in data and '"op":6' not in data:
return
back_to_json = json.loads(data)
if back_to_json["op"] == 2:
payload = back_to_json["d"]
inner_shard = payload.get("shard", [0])
self._identifies[inner_shard[0]].append(datetime.datetime.utcnow())
else:
self._resumes.append(datetime.datetime.utcnow())
self._clear_gateway_data()
def add_record(self, record: LogRecord):
self._gateway_queue.put_nowait(record)
async def notify_gateway_status(self, record: LogRecord):
types = {"INFO": ":information_source:", "WARNING": ":warning:"}
emoji = types.get(record.levelname, ":heavy_multiplication_x:")
dt = datetime.datetime.utcfromtimestamp(record.created)
msg = f"{emoji} `[{dt:%Y-%m-%d %H:%M:%S}] {record.message}`"
await self.webhook("gateway").send(msg)
@command_extra(name="commandstats", hidden=True, deletable=True)
@commands.is_owner()
async def _commandstats(self, ctx: ContextPlus, limit=20):
counter = self.bot.stats["commands"]
width = len(max(counter, key=len)) + 1
if limit > 0:
common = counter.most_common(limit)
else:
common = counter.most_common()[limit:]
output = "\n".join(f"{k:<{width}}: {c}" for k, c in common)
await ctx.send(f"```\n{output}\n```")
@command_extra(name="socketstats", hidden=True, deletable=True)
@commands.is_owner()
async def _socketstats(self, ctx: ContextPlus):
delta = datetime.datetime.now() - self.bot.uptime
minutes = delta.total_seconds() / 60
counter = self.bot.stats["socket"]
if None in counter:
counter.pop(None)
width = len(max(counter, key=len)) + 1
common = counter.most_common()
total = sum(self.bot.stats["socket"].values())
cpm = total / minutes
output = "\n".join(f"{k:<{width}}: {c}" for k, c in common)
await ctx.send(
f"{total} socket events observed ({cpm:.2f}/minute):"
f"```\n{output}\n```"
)
@command_extra(name="uptime")
async def _uptime(self, ctx: ContextPlus):
uptime = humanize.naturaltime(
datetime.datetime.now() - self.bot.uptime
)
await ctx.send(f"Uptime: **{uptime}**")
async def on_error(self, event, *args):
e = discord.Embed(title="Event Error", colour=0xA32952)
e.add_field(name="Event", value=event)
e.description = f"```py\n{traceback.format_exc()}\n```"
e.timestamp = datetime.datetime.utcnow()
args_str = ["```py"]
for index, arg in enumerate(args):
args_str.append(f"[{index}]: {arg!r}")
args_str.append("```")
e.add_field(name="Args", value="\n".join(args_str), inline=False)
hook = self.get_cog("Logs").webhook("errors")
try:
await hook.send(embed=e)
except (
discord.HTTPException,
discord.NotFound,
discord.Forbidden,
discord.InvalidArgument,
):
pass

View file

@ -1,8 +1,10 @@
import asyncio
import datetime
import logging
from collections import Counter
from typing import List, Union
import aiohttp
import discord
from discord.ext import commands
from rich import box
@ -33,7 +35,7 @@ log = logging.getLogger("tuxbot")
console = Console()
install(console=console)
packages: List[str] = ["jishaku", "tuxbot.cogs.admin"]
packages: List[str] = ["jishaku", "tuxbot.cogs.admin", "tuxbot.cogs.logs"]
class Tux(commands.AutoShardedBot):
@ -55,6 +57,10 @@ class Tux(commands.AutoShardedBot):
self.last_exception = None
self.logs = logs_data_path(self.instance_name)
self.console = console
self.stats = {"commands": Counter(), "socket": Counter()}
self.config: Config = ConfigFile(
str(data_path(self.instance_name) / "config.yaml"), Config
).config
@ -84,6 +90,7 @@ class Tux(commands.AutoShardedBot):
self._app_owners_fetched = False # to prevent abusive API calls
super().__init__(*args, help_command=None, **kwargs)
self.session = aiohttp.ClientSession(loop=self.loop)
async def load_packages(self):
if packages:

View file

@ -1,5 +1,5 @@
import logging
from typing import List, Dict, Any, NoReturn
from typing import List, Dict, Any
from structured_config import (
Structure,
IntField,
@ -80,7 +80,7 @@ def search_for(config, key, value, default=False) -> Any:
return default
def set_for_key(config, key, ctype, **values) -> NoReturn:
def set_for_key(config, key, ctype, **values) -> None:
# pylint: disable=anomalous-backslash-in-string
"""
La fonction suivante \`*-.
@ -105,6 +105,6 @@ def set_for_key(config, key, ctype, **values) -> NoReturn:
setattr(config[key], k, v)
def set_for(config, **values) -> NoReturn:
def set_for(config, **values) -> None:
for k, v in values.items():
setattr(config, k, v)

View file

@ -12,24 +12,45 @@ TOKEN_REPLACEMENT = "whoops, leaked token"
class ContextPlus(commands.Context):
async def send(self, *args, content=None, **kwargs):
if content is not None:
async def send(
self,
content=None,
*,
tts=False,
embed=None,
file=None,
files=None,
delete_after=None,
nonce=None,
allowed_mentions=None,
deletable=False
): # i know *args and **kwargs but, i prefer work with same values
if content:
content = content.replace(
self.bot.config.Core.token, TOKEN_REPLACEMENT
)
if kwargs.get("embed"):
embed = kwargs["embed"].to_dict()
for key, value in embed.items():
if embed:
e = embed.to_dict()
for key, value in e.items():
if isinstance(value, (str, bytes)):
embed[key] = value.replace(
e[key] = value.replace(
self.bot.config.Core.token, TOKEN_REPLACEMENT
)
kwargs["embed"] = Embed.from_dict(embed)
embed = Embed.from_dict(e)
if (
hasattr(self.command, "deletable") and self.command.deletable
) or kwargs.pop("deletable", False):
message = await super().send(content, *args, **kwargs)
) or deletable:
message = await super().send(
content=content,
tts=tts,
embed=embed,
file=file,
files=files,
delete_after=delete_after,
nonce=nonce,
allowed_mentions=allowed_mentions,
)
await message.add_reaction("🗑")
def check(reaction: discord.Reaction, user: discord.User):
@ -49,7 +70,16 @@ class ContextPlus(commands.Context):
await message.delete()
return message
return await super().send(content, *args, **kwargs)
return await super().send(
content=content,
tts=tts,
embed=embed,
file=file,
files=files,
delete_after=delete_after,
nonce=nonce,
allowed_mentions=allowed_mentions,
)
class CommandPLus(flags.FlagCommand):

View file

@ -24,19 +24,19 @@ def init_logging(level: int, location: pathlib.Path) -> None:
Where to store logs.
"""
dpy_logger = logging.getLogger("discord")
dpy_logger.setLevel(logging.WARN)
dpy_logger_file = location / "discord.log"
# dpy_logger = logging.getLogger("discord")
# dpy_logger.setLevel(logging.WARN)
# dpy_logger_file = location / "discord.log"
base_logger = logging.getLogger("tuxbot")
base_logger.setLevel(level)
base_logger_file = location / "tuxbot.log"
dpy_handler = logging.handlers.RotatingFileHandler(
str(dpy_logger_file.resolve()),
maxBytes=MAX_BYTES,
backupCount=MAX_OLD_LOGS,
)
# dpy_handler = logging.handlers.RotatingFileHandler(
# str(dpy_logger_file.resolve()),
# maxBytes=MAX_BYTES,
# backupCount=MAX_OLD_LOGS,
# )
base_handler = logging.handlers.RotatingFileHandler(
str(base_logger_file.resolve()),
maxBytes=MAX_BYTES,
@ -46,8 +46,8 @@ def init_logging(level: int, location: pathlib.Path) -> None:
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setFormatter(formatter)
dpy_handler.setFormatter(formatter)
# dpy_handler.setFormatter(formatter)
base_handler.setFormatter(formatter)
dpy_logger.addHandler(dpy_handler)
# dpy_logger.addHandler(dpy_handler)
base_logger.addHandler(base_handler)

View file

@ -5,7 +5,7 @@ import re
import sys
from argparse import Namespace
from pathlib import Path
from typing import NoReturn, Union, List
from typing import Union, List
from rich.prompt import Prompt, IntPrompt
from rich.console import Console
@ -263,7 +263,7 @@ def additional_config(instance: str, cogs: str = "**"):
)
def finish_setup(data_dir: Path) -> NoReturn:
def finish_setup(data_dir: Path) -> None:
"""Configs who directly refer to the bot.
Parameters
@ -311,7 +311,7 @@ def finish_setup(data_dir: Path) -> NoReturn:
instance_config.config.Core.locale = "en-US"
def basic_setup() -> NoReturn:
def basic_setup() -> None:
"""Configs who refer to instances."""
console.print(
Rule(
@ -395,7 +395,7 @@ def parse_cli_flags(args: list) -> Namespace:
return args
def setup() -> NoReturn:
def setup() -> None:
cli_flags = parse_cli_flags(sys.argv[1:])
try: