feat(botCore): start core template

.idea/workspace.xml

@@ -2,35 +2,11 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="c97c8a30-7573-4dcd-a0d4-5bf94b8ddbbd" name="5ed57ed9960f35191182a924 core" comment="">
-      <change beforePath="$PROJECT_DIR$/.github/issue_template.md" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/.gitignore" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/tuxbot/__main__.py" beforeDir="false" afterPath="$PROJECT_DIR$/tuxbot/__main__.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/dictionaries/romain.xml" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/tuxbot/core/bot.py" beforeDir="false" afterPath="$PROJECT_DIR$/tuxbot/core/bot.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/discord.xml" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/inspectionProfiles/Project_Default.xml" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/inspectionProfiles/profiles_settings.xml" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/misc.xml" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/modules.xml" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/tuxbot-bot-rewrite.iml" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/vcs.xml" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/setup.cfg" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/setup.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/__init__.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/__main__.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/cogs/images/__init__.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/cogs/images/images.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/cogs/logs/__init__.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/cogs/logs/logs.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/cogs/network/__init__.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/cogs/network/network.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/core/__init__.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/core/bot.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/core/config.py" beforeDir="false" />
       <change beforePath="$PROJECT_DIR$/tuxbot/core/models/__init__.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/core/utils/functions/cli.py" beforeDir="false" />
+      <change beforePath="$PROJECT_DIR$/tuxbot/setup.py" beforeDir="false" afterPath="$PROJECT_DIR$/tuxbot/setup.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/core/utils/functions/extra.py" beforeDir="false" />
-      <change beforePath="$PROJECT_DIR$/tuxbot/setup.py" beforeDir="false" />
     </list>
     <list id="a3abf5c0-7587-46e4-8f09-88e34a1ab8a4" name="5ed41911b012e33f68a07e7a i18n" comment="" />
     <list id="6566fca1-2e90-48bb-9e74-dd3badbaca99" name="Default Changelist" comment="" />
@@ -62,7 +38,7 @@
     <property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
     <property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
     <property name="WebServerToolWindowFactoryState" value="false" />
-    <property name="last_opened_file_path" value="$PROJECT_DIR$/tuxbot/cogs/network" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$/tuxbot" />
     <property name="node.js.detected.package.eslint" value="true" />
     <property name="node.js.detected.package.tslint" value="true" />
     <property name="node.js.path.for.package.eslint" value="project" />
@@ -81,11 +57,11 @@
       <recent name="$PROJECT_DIR$/tuxbot/core" />
     </key>
     <key name="CopyFile.RECENT_KEYS">
+      <recent name="$PROJECT_DIR$/tuxbot" />
       <recent name="$PROJECT_DIR$/tuxbot/cogs/network" />
       <recent name="$PROJECT_DIR$" />
       <recent name="$PROJECT_DIR$/tuxbot/cogs" />
       <recent name="$PROJECT_DIR$/utils/locales" />
-      <recent name="$PROJECT_DIR$/cogs" />
     </key>
   </component>
   <component name="SvnConfiguration">
@@ -120,7 +96,7 @@
     </task>
     <task id="5ed41911b012e33f68a07e7a" summary="i18n">
       <changelist id="a3abf5c0-7587-46e4-8f09-88e34a1ab8a4" name="5ed41911b012e33f68a07e7a i18n" comment="" />
-      <created>1591200420454</created>
+      <created>1591205009488</created>
       <option name="issue" value="true" />
       <url>https://trello.com/c/vK0cBbF2/38-i18n</url>
       <option name="number" value="38" />
@@ -132,7 +108,7 @@
     </task>
     <task active="true" id="5ed57ed9960f35191182a924" summary="core">
       <changelist id="c97c8a30-7573-4dcd-a0d4-5bf94b8ddbbd" name="5ed57ed9960f35191182a924 core" comment="" />
-      <created>1591200420454</created>
+      <created>1591205009488</created>
       <option name="issue" value="true" />
       <url>https://trello.com/c/SafaMBht/40-core</url>
       <option name="number" value="40" />
@@ -141,7 +117,7 @@
       <workItem from="1591049956280" duration="4910000" />
       <workItem from="1591054878071" duration="1039000" />
       <workItem from="1591088657371" duration="4107000" />
-      <workItem from="1591128560850" duration="24277000" />
+      <workItem from="1591128560850" duration="29106000" />
     </task>
     <option name="localTasksCounter" value="2" />
     <option name="createBranch" value="false" />

tuxbot/__main__.py

@@ -1,2 +1,138 @@
+import argparse
+import asyncio
+import logging
+import signal
+import sys
+
+import discord
+from colorama import Fore, init, Style
+
+import tuxbot.logging
+from tuxbot.core import data_manager
+
+log = logging.getLogger("tuxbot.main")
+init()
+
+
+def parse_cli_flags(args):
+    parser = argparse.ArgumentParser(
+        description="Tuxbot - OpenSource bot",
+        usage="tuxbot <instance_name> [arguments]"
+    )
+    parser.add_argument("--version", "-V", help="Show tuxbot's used version")
+    parser.add_argument("--list-instances", "-L",
+                        help="List all instance names")
+    parser.add_argument(
+        "instance_name", nargs="?",
+        help="Name of the bot instance created during `redbot-setup`.")
+
+    args = parser.parse_args(args)
+
+    if args.prefix:
+        args.prefix = sorted(args.prefix, reverse=True)
+    else:
+        args.prefix = []
+
+    return args
+
+
+async def shutdown_handler(tux, signal_type, exit_code=None):
+    if signal_type:
+        log.info("%s received. Quitting...", signal_type)
+        sys.exit(0)
+    elif exit_code is None:
+        log.info("Shutting down from unhandled exception")
+        tux._shutdown_mode = 1
+
+    if exit_code is not None:
+        tux._shutdown_mode = exit_code
+
+    try:
+        await tux.logout()
+    finally:
+        pending = [
+            t for t in asyncio.all_tasks() if t is not asyncio.current_task()
+        ]
+
+        for task in pending:
+            task.cancel()
+
+        await asyncio.gather(*pending, return_exceptions=True)
+
+
+async def run_bot(tux: Tuxbot, cli_flags: argparse.Namespace) -> None:
+    data_path = data_manager.get_data_path(tuxbot.instance_name)
+
+    tuxbot.logging.init_logging(
+        level=cli_flags.logging_level,
+        location=data_path / "logs"
+    )
+
+    log.debug("====Basic Config====")
+    log.debug("Data Path: %s", data_path)
+
+    if cli_flags.token:
+        token = cli_flags.token
+    else:
+        token = await tux._config.token()
+
+    if not token:
+        log.critical("Token must be set if you want to login.")
+        sys.exit(1)
+
+    try:
+        await tux.start(token, bot=True, cli_flags=cli_flags)
+    except discord.LoginFailure:
+        log.critical("This token appears to be valid.")
+        sys.exit(1)
+
+    return None
+
+
 def main():
-    ...
+    tux = None
+    cli_flags = parse_cli_flags(sys.argv[1:])
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+
+    try:
+        if cli_flags.no_instance:
+            print(Fore.RED
+                  + "No instance provided ! "
+                    "You can use 'tuxbot -L' to list all available instances"
+                  + Style.RESET_ALL)
+            sys.exit(1)
+
+        tux = Tuxbot(
+            cli_flags=cli_flags,
+            description="Tuxbot, made from and for OpenSource",
+            dm_help=None
+        )
+
+        loop.run_until_complete(run_bot(tux, cli_flags))
+    except KeyboardInterrupt:
+        log.warning("Please use <prefix>quit instead of Ctrl+C to Shutdown!")
+        log.error("Received KeyboardInterrupt")
+        if tuxbot is not None:
+            loop.run_until_complete(shutdown_handler(tux, signal.SIGINT))
+    except SystemExit as exc:
+        log.info("Shutting down with exit code: %s", exc.code)
+        if tuxbot is not None:
+            loop.run_until_complete(shutdown_handler(tux, None, exc.code))
+    except Exception as exc:
+        log.exception("Unexpected exception (%s): ", type(exc), exc_info=exc)
+        if tuxbot is not None:
+            loop.run_until_complete(shutdown_handler(tux, None, 1))
+    finally:
+        loop.run_until_complete(loop.shutdown_asyncgens())
+        log.info("Please wait, cleaning up a bit more")
+        loop.run_until_complete(asyncio.sleep(2))
+        asyncio.set_event_loop(None)
+        loop.stop()
+        loop.close()
+        exit_code = 1 if tuxbot is None else tux._shutdown_mode
+        sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()

tuxbot/core/bot.py

@@ -8,10 +8,15 @@ __all__ = ["Tux"]
 
 
 class Tux(commands.AutoShardedBot):
-    def __init__(self, *args, bot_dir: Path, **kwargs):
+    def __init__(self, *args, cli_flags=None, bot_dir: Path = Path.cwd(), **kwargs):
+        # by default, if the bot shutdown without any intervention,
+        # it's a crash
+        self._shutdown_mode = 1
+        self._cli_flags = cli_flags
+        self._last_exception = None
+
         self._config = Config.register_core(
-            identifier=None,
+            identifier=self._cli_flags.instance_name
-            mentionnable=False
         )
         self._config.register_global(
             token=None,
@@ -22,7 +27,6 @@ class Tux(commands.AutoShardedBot):
             locale="en-US",
             embeds=True,
             color=0x6E83D1,
-            description="Tuxbot !",
             disabled_commands=[]
         )
         self._config.register_guild(
@@ -38,4 +42,18 @@ class Tux(commands.AutoShardedBot):
         )
         self._config.register_channel(
             ignored=False
-        )
+        )
+
+        if "owner_ids" in kwargs:
+            kwargs["owner_ids"] = set(kwargs["owner_ids"])
+        else:
+            kwargs["owner_ids"] = self._config.owner_ids()
+
+        message_cache_size = 100_000
+        kwargs["max_messages"] = message_cache_size
+        self._max_messages = message_cache_size
+
+        self._uptime = None
+        self._main_dir = bot_dir
+
+        super().__init__(*args, help_command=None, **kwargs)

tuxbot/core/data_manager.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+import appdirs
+
+app_dir = appdirs.AppDirs("Tuxbot-bot")
+config_dir = Path(app_dir.user_config_dir)
+config_file = config_dir / "config.json"
+
+
+def get_data_path(instance_name: str) -> Path:
+    return Path(app_dir.user_data_dir) / "data" / instance_name
+
+
+def get_core_path(instance_name: str) -> Path:
+    data_path = get_data_path(instance_name)
+    return data_path / "data" / instance_name / "core"
+
+
+def get_cogs_path(instance_name: str) -> Path:
+    data_path = get_data_path(instance_name)
+    return data_path / "data" / instance_name / "cogs"
+
+
+def get_cog_path(instance_name: str, cog_name: str) -> Path:
+    data_path = get_data_path(instance_name)
+    return data_path / "data" / instance_name / "cogs" / cog_name

tuxbot/logging.py

@@ -0,0 +1,151 @@
+import logging.handlers
+import pathlib
+import re
+import sys
+from typing import List, Tuple, Optional
+
+MAX_OLD_LOGS = 8
+
+
+class RotatingFileHandler(logging.handlers.RotatingFileHandler):
+    """Custom rotating file handler.
+
+    This file handler rotates a bit differently to the one in stdlib.
+
+    For a start, this works off of a "stem" and a "directory". The stem
+    is the base name of the log file, without the extension. The
+    directory is where all log files (including backups) will be placed.
+
+    Secondly, this logger rotates files downwards, and new logs are
+    *started* with the backup number incremented. The stdlib handler
+    rotates files upwards, and this leaves the logs in reverse order.
+
+    Thirdly, naming conventions are not customisable with this class.
+    Logs will initially be named in the format "{stem}.log", and after
+    rotating, the first log file will be renamed "{stem}-part1.log",
+    and a new file "{stem}-part2.log" will be created for logging to
+    continue.
+
+    A few things can't be modified in this handler: it must use append
+    mode, it doesn't support use of the `delay` arg, and it will ignore
+    custom namers and rotators.
+
+    When this handler is instantiated, it will search through the
+    directory for logs from previous runtimes, and will open the file
+    with the highest backup number to append to.
+    """
+
+    def __init__(
+        self,
+        stem: str,
+        directory: pathlib.Path,
+        maxBytes: int = 0,
+        backupCount: int = 0,
+        encoding: Optional[str] = None,
+    ) -> None:
+        self.baseStem = stem
+        self.directory = directory.resolve()
+        # Scan for existing files in directory, append to last part of existing log
+        log_part_re = re.compile(rf"{stem}-part(?P<partnum>\d+).log")
+        highest_part = 0
+        for path in directory.iterdir():
+            match = log_part_re.match(path.name)
+            if match and int(match["partnum"]) > highest_part:
+                highest_part = int(match["partnum"])
+        if highest_part:
+            filename = directory / f"{stem}-part{highest_part}.log"
+        else:
+            filename = directory / f"{stem}.log"
+        super().__init__(
+            filename,
+            mode="a",
+            maxBytes=maxBytes,
+            backupCount=backupCount,
+            encoding=encoding,
+            delay=False,
+        )
+
+    def doRollover(self):
+        if self.stream:
+            self.stream.close()
+            self.stream = None
+        initial_path = self.directory / f"{self.baseStem}.log"
+        if self.backupCount > 0 and initial_path.exists():
+            initial_path.replace(self.directory / f"{self.baseStem}-part1.log")
+
+        match = re.match(
+            rf"{self.baseStem}(?:-part(?P<part>\d+)?)?.log", pathlib.Path(self.baseFilename).name
+        )
+        latest_part_num = int(match.groupdict(default="1").get("part", "1"))
+        if self.backupCount < 1:
+            # No backups, just delete the existing log and start again
+            pathlib.Path(self.baseFilename).unlink()
+        elif latest_part_num > self.backupCount:
+            # Rotate files down one
+            # red-part2.log becomes red-part1.log etc, a new log is added at the end.
+            for i in range(1, self.backupCount):
+                next_log = self.directory / f"{self.baseStem}-part{i + 1}.log"
+                if next_log.exists():
+                    prev_log = self.directory / f"{self.baseStem}-part{i}.log"
+                    next_log.replace(prev_log)
+        else:
+            # Simply start a new file
+            self.baseFilename = str(
+                self.directory / f"{self.baseStem}-part{latest_part_num + 1}.log"
+            )
+
+        self.stream = self._open()
+
+
+def init_logging(level: int, location: pathlib.Path) -> None:
+    dpy_logger = logging.getLogger("discord")
+    dpy_logger.setLevel(logging.WARNING)
+    base_logger = logging.getLogger("red")
+    base_logger.setLevel(level)
+
+    formatter = logging.Formatter(
+        "[{asctime}] [{levelname}] {name}: {message}", datefmt="%Y-%m-%d %H:%M:%S", style="{"
+    )
+
+    stdout_handler = logging.StreamHandler(sys.stdout)
+    stdout_handler.setFormatter(formatter)
+    base_logger.addHandler(stdout_handler)
+    dpy_logger.addHandler(stdout_handler)
+
+    if not location.exists():
+        location.mkdir(parents=True, exist_ok=True)
+    # Rotate latest logs to previous logs
+    previous_logs: List[pathlib.Path] = []
+    latest_logs: List[Tuple[pathlib.Path, str]] = []
+    for path in location.iterdir():
+        match = re.match(r"latest(?P<part>-part\d+)?\.log", path.name)
+        if match:
+            part = match.groupdict(default="")["part"]
+            latest_logs.append((path, part))
+        match = re.match(r"previous(?:-part\d+)?.log", path.name)
+        if match:
+            previous_logs.append(path)
+    # Delete all previous.log files
+    for path in previous_logs:
+        path.unlink()
+    # Rename latest.log files to previous.log
+    for path, part in latest_logs:
+        path.replace(location / f"previous{part}.log")
+
+    latest_fhandler = RotatingFileHandler(
+        stem="latest",
+        directory=location,
+        maxBytes=1_000_000,  # About 1MB per logfile
+        backupCount=MAX_OLD_LOGS,
+        encoding="utf-8",
+    )
+    all_fhandler = RotatingFileHandler(
+        stem="red",
+        directory=location,
+        maxBytes=1_000_000,
+        backupCount=MAX_OLD_LOGS,
+        encoding="utf-8",
+    )
+    for fhandler in (latest_fhandler, all_fhandler):
+        fhandler.setFormatter(formatter)
+        base_logger.addHandler(fhandler)

tuxbot/setup.py

@@ -1,19 +1,16 @@
 import json
 import logging
-import os
 import re
 import sys
 from pathlib import Path
-from typing import NoReturn, Union, List
+from typing import NoReturn, Union, List, Set
 
-import appdirs
 import click
 from colorama import Fore, Style, init
 
-init()
+from tuxbot.core.data_manager import config_dir, app_dir
 
-app_dir = appdirs.AppDirs("Tuxbot-bot")
+init()
-config_dir = Path(app_dir.user_config_dir)
 
 try:
     config_dir.mkdir(parents=True, exist_ok=True)
@@ -133,6 +130,7 @@ def get_data_dir(instance_name: str) -> Path:
 
     (data_path_input / 'core').mkdir(parents=True, exist_ok=True)
     (data_path_input / 'cogs').mkdir(parents=True, exist_ok=True)
+    (data_path_input / 'logs').mkdir(parents=True, exist_ok=True)
 
     return data_path_input
 
@@ -156,14 +154,15 @@ def get_token() -> str:
     return token
 
 
-def get_prefixes() -> List[str]:
+def get_multiple(question: str, confirmation: str, value_type: type)\
-    print("Choice a (or multiple) prefix for the bot")
+        -> Set[Union[str, int]]:
-    prefixes = [input('> ')]
+    print(question)
+    values = [value_type(input('> '))]
 
-    while click.confirm("Add another prefix ?", default=False):
+    while click.confirm(confirmation, default=False):
-        prefixes.append(input('> '))
+        values.append(value_type(input('> ')))
 
-    return prefixes
+    return set(values)
 
 
 def additional_config() -> dict:
@@ -193,8 +192,21 @@ def finish_setup(data_dir: Path) -> NoReturn:
 
     token = get_token()
     print()
-    prefixes = get_prefixes()
+    prefixes = get_multiple(
-    mentionable = click.confirm("Does the bot answer if it's mentioned?", default=True)
+        "Choice a (or multiple) prefix for the bot",
+        "Add another prefix ?",
+        str
+    )
+    mentionable = click.confirm(
+        "Does the bot answer if it's mentioned?",
+        default=True
+    )
+
+    owners_id = get_multiple(
+        "Give the owner id of this bot",
+        "Add another owner ?",
+        int
+    )
 
     cogs_config = additional_config()
 
@@ -203,6 +215,7 @@ def finish_setup(data_dir: Path) -> NoReturn:
         'token': token,
         'prefixes': prefixes,
         'mentionable': mentionable,
+        'owners_id': owners_id,
     }
 
     with core_file.open("w") as fs:
@@ -260,7 +273,7 @@ def setup():
     try:
         """Create a new instance."""
         level = logging.DEBUG
-        base_logger = logging.getLogger("tux")
+        base_logger = logging.getLogger("tuxbot")
         base_logger.setLevel(level)
         formatter = logging.Formatter(
             "[{asctime}] [{levelname}] {name}: {message}",