Merge branch 'starlette_cleanup' into develop

core/__init__.py

@@ -1,6 +1,6 @@
 from core.config import CoreConfig
-from core.allnet import AllnetServlet
+from core.allnet import AllnetServlet, BillingServlet
-from core.aimedb import AimedbFactory
+from core.aimedb import AimedbServlette
 from core.title import TitleServlet
 from core.utils import Utils
 from core.mucha import MuchaServlet

core/adb_handlers/base.py

@@ -102,7 +102,7 @@ class ADBHeader:
         magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id = struct.unpack_from("<5H6sI12s", data)
         head = cls(magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id)
         
-        if head.length != len(data):
+        if head.length > len(data):
             raise ADBHeaderException(f"Length is incorrect! Expect {head.length}, got {len(data)}")
         
         return head

core/aimedb.py

@@ -1,9 +1,7 @@
-from twisted.internet.protocol import Factory, Protocol
 import logging, coloredlogs
 from Crypto.Cipher import AES
-import struct
+from typing import Dict, Tuple, Callable, Union, Optional
-from typing import Dict, Tuple, Callable, Union
+import asyncio
-from typing_extensions import Final
 from logging.handlers import TimedRotatingFileHandler
 
 from core.config import CoreConfig
@@ -11,15 +9,37 @@ from core.utils import create_sega_auth_key
 from core.data import Data
 from .adb_handlers import *
 
-
+class AimedbServlette():
-class AimedbProtocol(Protocol):
     request_list: Dict[int, Tuple[Callable[[bytes, int], Union[ADBBaseResponse, bytes]], int, str]] = {}
-
     def __init__(self, core_cfg: CoreConfig) -> None:        
-        self.logger = logging.getLogger("aimedb")
         self.config = core_cfg        
         self.data = Data(core_cfg)
-        if core_cfg.aimedb.key == "":
+
+        self.logger = logging.getLogger("aimedb")
+        if not hasattr(self.logger, "initted"):
+            log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "aimedb"),
+                when="d",
+                backupCount=10,
+            )
+            fileHandler.setFormatter(log_fmt)
+
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
+
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(self.config.aimedb.loglevel)
+            coloredlogs.install(
+                level=core_cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.initted = True
+
+        if not core_cfg.aimedb.key:
             self.logger.error("!!!KEY NOT SET!!!")
             exit(1)
 
@@ -40,27 +60,30 @@ class AimedbProtocol(Protocol):
 
         self.register_handler(0x13, 0x14, self.handle_log_ex, 'aime_log_ex')
         self.register_handler(0x64, 0x65, self.handle_hello, 'hello')
-        self.register_handler(0x66, 0, self.handle_goodbye, 'goodbye')
 
     def register_handler(self, cmd: int, resp:int, handler: Callable[[bytes, int], Union[ADBBaseResponse, bytes]], name: str) -> None:
         self.request_list[cmd] = (handler, resp, name)
     
-    def append_padding(self, data: bytes):
+    def start(self) -> None:
-        """Appends 0s to the end of the data until it's at the correct size"""
+        self.logger.info(f"Start on port {self.config.aimedb.port}")
-        length = struct.unpack_from("<H", data, 6)
+        asyncio.create_task(asyncio.start_server(self.dataReceived, self.config.server.listen_address, self.config.aimedb.port))
-        padding_size = length[0] - len(data)
-        data += bytes(padding_size)
-        return data
     
-    def connectionMade(self) -> None:
+    async def dataReceived(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
-        self.logger.debug(f"{self.transport.getPeer().host} Connected")
+        self.logger.debug(f"Connection made from {writer.get_extra_info('peername')[0]}")
+        while True:
+            try:
+                data: bytes = await reader.read(4096)
+                if len(data) == 0:
+                    self.logger.debug("Connection closed")
+                    return
+                await self.process_data(data, reader, writer)
+                await writer.drain()
+            except ConnectionResetError as e:
+                self.logger.debug("Connection reset, disconnecting")
+                return
 
-    def connectionLost(self, reason) -> None:
+    async def process_data(self, data: bytes, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> Optional[bytes]:
-        self.logger.debug(
+        addr = writer.get_extra_info('peername')[0]
-            f"{self.transport.getPeer().host} Disconnected - {reason.value}"
-        )
-
-    def dataReceived(self, data: bytes) -> None:
         cipher = AES.new(self.config.aimedb.key.encode(), AES.MODE_ECB)
 
         try:
@@ -68,9 +91,9 @@ class AimedbProtocol(Protocol):
         
         except Exception as e:
             self.logger.error(f"Failed to decrypt {data.hex()} because {e}")
-            return None
+            return
 
-        self.logger.debug(f"{self.transport.getPeer().host} wrote {decrypted.hex()}")
+        self.logger.debug(f"{addr} wrote {decrypted.hex()}")
 
         try:
             head = ADBHeader.from_data(decrypted)
@@ -79,7 +102,9 @@ class AimedbProtocol(Protocol):
             self.logger.error(f"Error parsing ADB header: {e}")
             try:    
                 encrypted = cipher.encrypt(ADBBaseResponse().make())
-                self.transport.write(encrypted)
+                writer.write(encrypted)
+                await writer.drain()
+                return
 
             except Exception as e:
                 self.logger.error(f"Failed to encrypt default response because {e}")
@@ -89,46 +114,51 @@ class AimedbProtocol(Protocol):
         if head.keychip_id == "ABCD1234567" or head.store_id == 0xfff0:
             self.logger.warning(f"Request from uninitialized AMLib: {vars(head)}")
 
+        if head.cmd == 0x66:
+            self.logger.info("Goodbye")
+            writer.close()
+            return
+
         handler, resp_code, name = self.request_list.get(head.cmd, (self.handle_default, None, 'default'))
 
         if resp_code is None:
             self.logger.warning(f"No handler for cmd {hex(head.cmd)}")
         
         elif resp_code > 0:
-            self.logger.info(f"{name} from {head.keychip_id} ({head.game_id}) @ {self.transport.getPeer().host}")
+            self.logger.info(f"{name} from {head.keychip_id} ({head.game_id}) @ {addr}")
         
-        resp = handler(decrypted, resp_code)
+        resp = await handler(decrypted, resp_code)
 
         if type(resp) == ADBBaseResponse or issubclass(type(resp), ADBBaseResponse):
             resp_bytes = resp.make()
-            if len(resp_bytes) != resp.head.length:
-                resp_bytes = self.append_padding(resp_bytes)
 
         elif type(resp) == bytes:
             resp_bytes = resp
         
         elif resp is None: # Nothing to send, probably a goodbye
+            self.logger.warn(f"None return by handler for {name}")
             return
         
         else:
+            self.logger.error(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
             raise TypeError(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
 
         try:
             encrypted = cipher.encrypt(resp_bytes)
             self.logger.debug(f"Response {resp_bytes.hex()}")
-            self.transport.write(encrypted)
+            writer.write(encrypted)
 
         except Exception as e:
             self.logger.error(f"Failed to encrypt {resp_bytes.hex()} because {e}")
     
-    def handle_default(self, data: bytes, resp_code: int, length: int = 0x20) -> ADBBaseResponse:
+    async def handle_default(self, data: bytes, resp_code: int, length: int = 0x20) -> ADBBaseResponse:
         req = ADBHeader.from_data(data)
         return ADBBaseResponse(resp_code, length, 1, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
 
-    def handle_hello(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_hello(self, data: bytes, resp_code: int) -> ADBBaseResponse:
-        return self.handle_default(data, resp_code)
+        return await self.handle_default(data, resp_code)
 
-    def handle_campaign(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_campaign(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         h = ADBHeader.from_data(data)
         if h.protocol_ver >= 0x3030:
             req = h
@@ -143,11 +173,11 @@ class AimedbProtocol(Protocol):
         # We don't currently support campaigns
         return resp
 
-    def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         req = ADBLookupRequest(data)
-        user_id = self.data.card.get_user_id_from_card(req.access_code)
+        user_id = await self.data.card.get_user_id_from_card(req.access_code)
-        is_banned = self.data.card.get_card_banned(req.access_code)
+        is_banned = await self.data.card.get_card_banned(req.access_code)
-        is_locked = self.data.card.get_card_locked(req.access_code)
+        is_locked = await self.data.card.get_card_locked(req.access_code)
         
         ret = ADBLookupResponse.from_req(req.head, user_id)
         if is_banned and is_locked:
@@ -162,12 +192,12 @@ class AimedbProtocol(Protocol):
         )
         return ret
 
-    def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         req = ADBLookupRequest(data)
-        user_id = self.data.card.get_user_id_from_card(req.access_code)
+        user_id = await self.data.card.get_user_id_from_card(req.access_code)
 
-        is_banned = self.data.card.get_card_banned(req.access_code)
+        is_banned = await self.data.card.get_card_banned(req.access_code)
-        is_locked = self.data.card.get_card_locked(req.access_code)
+        is_locked = await self.data.card.get_card_locked(req.access_code)
 
         ret = ADBLookupExResponse.from_req(req.head, user_id)
         if is_banned and is_locked:
@@ -191,7 +221,7 @@ class AimedbProtocol(Protocol):
 
         return ret
 
-    def handle_felica_lookup(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_felica_lookup(self, data: bytes, resp_code: int) -> bytes:
         """
         On official, I think a card has to be registered for this to actually work, but 
         I'm making the executive decision to not implement that and just kick back our
@@ -207,7 +237,7 @@ class AimedbProtocol(Protocol):
         )
         return ADBFelicaLookupResponse.from_req(req.head, ac)
 
-    def handle_felica_register(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_felica_register(self, data: bytes, resp_code: int) -> bytes:
         """
         I've never seen this used.
         """
@@ -215,14 +245,14 @@ class AimedbProtocol(Protocol):
         ac = self.data.card.to_access_code(req.idm)
         
         if self.config.server.allow_user_registration:
-            user_id = self.data.user.create_user()
+            user_id = await self.data.user.create_user()
 
             if user_id is None:
                 self.logger.error("Failed to register user!")
                 user_id = -1
 
             else:
-                card_id = self.data.card.create_card(user_id, ac)
+                card_id = await self.data.card.create_card(user_id, ac)
 
                 if card_id is None:
                     self.logger.error("Failed to register card!")
@@ -239,10 +269,10 @@ class AimedbProtocol(Protocol):
 
         return ADBFelicaLookupResponse.from_req(req.head, ac)
 
-    def handle_felica_lookup_ex(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_felica_lookup_ex(self, data: bytes, resp_code: int) -> bytes:
         req = ADBFelicaLookup2Request(data)
         access_code = self.data.card.to_access_code(req.idm)
-        user_id = self.data.card.get_user_id_from_card(access_code=access_code)
+        user_id = await self.data.card.get_user_id_from_card(access_code=access_code)
 
         if user_id is None:
             user_id = -1
@@ -263,7 +293,7 @@ class AimedbProtocol(Protocol):
 
         return resp
 
-    def handle_campaign_clear(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_campaign_clear(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         req = ADBCampaignClearRequest(data)
 
         resp = ADBCampaignClearResponse.from_req(req.head)
@@ -271,19 +301,19 @@ class AimedbProtocol(Protocol):
         # We don't support campaign stuff
         return resp
 
-    def handle_register(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_register(self, data: bytes, resp_code: int) -> bytes:
         req = ADBLookupRequest(data)
         user_id = -1
 
         if self.config.server.allow_user_registration:
-            user_id = self.data.user.create_user()
+            user_id = await self.data.user.create_user()
 
             if user_id is None:
                 self.logger.error("Failed to register user!")
                 user_id = -1
 
             else:
-                card_id = self.data.card.create_card(user_id, req.access_code)
+                card_id = await self.data.card.create_card(user_id, req.access_code)
 
                 if card_id is None:
                     self.logger.error("Failed to register card!")
@@ -305,17 +335,17 @@ class AimedbProtocol(Protocol):
         return resp
 
     # TODO: Save these in some capacity, as deemed relevant
-    def handle_status_log(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_status_log(self, data: bytes, resp_code: int) -> bytes:
         req = ADBStatusLogRequest(data)
         self.logger.info(f"User {req.aime_id} logged {req.status.name} event")
         return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
 
-    def handle_log(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_log(self, data: bytes, resp_code: int) -> bytes:
         req = ADBLogRequest(data)
         self.logger.info(f"User {req.aime_id} logged {req.status.name} event, credit_ct: {req.credit_ct} bet_ct: {req.bet_ct} won_ct: {req.won_ct}")
         return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
 
-    def handle_log_ex(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_log_ex(self, data: bytes, resp_code: int) -> bytes:
         req = ADBLogExRequest(data)
         strs = []
         self.logger.info(f"Recieved {req.num_logs} or {len(req.logs)} logs")
@@ -324,43 +354,3 @@ class AimedbProtocol(Protocol):
             self.logger.debug(f"User {req.logs[x].aime_id} logged {req.logs[x].status.name} event, credit_ct: {req.logs[x].credit_ct} bet_ct: {req.logs[x].bet_ct} won_ct: {req.logs[x].won_ct}")
         return ADBLogExResponse.from_req(req.head)
 
-    def handle_goodbye(self, data: bytes, resp_code: int) -> None:
-        self.logger.info(f"goodbye from {self.transport.getPeer().host}")
-        self.transport.loseConnection()
-        return
-
-class AimedbFactory(Factory):
-    protocol = AimedbProtocol
-
-    def __init__(self, cfg: CoreConfig) -> None:
-        self.config = cfg
-        log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
-        self.logger = logging.getLogger("aimedb")
-
-        fileHandler = TimedRotatingFileHandler(
-            "{0}/{1}.log".format(self.config.server.log_dir, "aimedb"),
-            when="d",
-            backupCount=10,
-        )
-        fileHandler.setFormatter(log_fmt)
-
-        consoleHandler = logging.StreamHandler()
-        consoleHandler.setFormatter(log_fmt)
-
-        self.logger.addHandler(fileHandler)
-        self.logger.addHandler(consoleHandler)
-
-        self.logger.setLevel(self.config.aimedb.loglevel)
-        coloredlogs.install(
-            level=cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str
-        )
-
-        if self.config.aimedb.key == "":
-            self.logger.error("Please set 'key' field in your config file.")
-            exit(1)
-
-        self.logger.info(f"Ready on port {self.config.aimedb.port}")
-
-    def buildProtocol(self, addr):
-        return AimedbProtocol(self.config)

core/allnet.py

@@ -1,20 +1,24 @@
-from typing import Dict, List, Any, Optional, Tuple, Union, Final
-import logging, coloredlogs
-from logging.handlers import TimedRotatingFileHandler
-from twisted.web.http import Request
-from datetime import datetime
 import pytz
 import base64
 import zlib
 import json
+import yaml
+import logging
+import coloredlogs
+import urllib.parse
+import math
+from typing import Dict, List, Any, Optional, Union, Final
+from logging.handlers import TimedRotatingFileHandler
+from starlette.requests import Request
+from starlette.responses import PlainTextResponse
+from starlette.applications import Starlette
+from starlette.routing import Route
+from datetime import datetime
 from enum import Enum
 from Crypto.PublicKey import RSA
 from Crypto.Hash import SHA
 from Crypto.Signature import PKCS1_v1_5
-from time import strptime
+from os import path, environ, mkdir, access, W_OK
-from os import path
-import urllib.parse
-import math
 
 from .config import CoreConfig
 from .utils import Utils
@@ -91,7 +95,6 @@ class DLI_STATUS(Enum):
 
 class AllnetServlet:
     def __init__(self, core_cfg: CoreConfig, cfg_folder: str):
-        super().__init__()
         self.config = core_cfg
         self.config_folder = cfg_folder
         self.data = Data(core_cfg)
@@ -120,25 +123,21 @@ class AllnetServlet:
             )
             self.logger.initialized = True
 
-        plugins = Utils.get_all_titles()
-
-        if len(plugins) == 0:
-            self.logger.error("No games detected!")
-
         self.logger.info(
-            f"Serving {len(TitleServlet.title_registry)} game codes port {core_cfg.allnet.port}"
+            f"Ready on port {self.config.allnet.port if self.config.allnet.standalone else self.config.server.port}"
         )
 
-    def handle_poweron(self, request: Request, _: Dict):
+    async def handle_poweron(self, request: Request):
         request_ip = Utils.get_ip_addr(request)
-        pragma_header = request.getHeader('Pragma')
+        pragma_header = request.headers.get('Pragma', "")
         is_dfi = pragma_header is not None and pragma_header == "DFI"
+        data = await request.body()
         
         try:
             if is_dfi:
-                req_urlencode = self.from_dfi(request.content.getvalue())
+                req_urlencode = self.from_dfi(data)
             else:
-                req_urlencode = request.content.getvalue().decode()
+                req_urlencode = data
 
             req_dict = self.allnet_req_to_dict(req_urlencode)
             if req_dict is None:
@@ -155,7 +154,7 @@ class AllnetServlet:
         except AllnetRequestException as e:
             if e.message != "":
                 self.logger.error(e)
-            return b""
+            return PlainTextResponse()
 
         if req.format_ver == 3:
             resp = AllnetPowerOnResponse3(req.token)
@@ -166,42 +165,42 @@ class AllnetServlet:
 
         self.logger.debug(f"Allnet request: {vars(req)}")
 
-        machine = self.data.arcade.get_machine(req.serial)        
+        machine = await self.data.arcade.get_machine(req.serial)        
         if machine is None and not self.config.server.allow_unregistered_serials:
             msg = f"Unrecognised serial {req.serial} attempted allnet auth from {request_ip}."
-            self.data.base.log_event(
+            await self.data.base.log_event(
                 "allnet", "ALLNET_AUTH_UNKNOWN_SERIAL", logging.WARN, msg
             )
             self.logger.warning(msg)
 
             resp.stat = ALLNET_STAT.bad_machine.value
             resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-            return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+            return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
         if machine is not None:
-            arcade = self.data.arcade.get_arcade(machine["arcade"])
+            arcade = await self.data.arcade.get_arcade(machine["arcade"])
             if self.config.server.check_arcade_ip:
                 if arcade["ip"] and arcade["ip"] is not None and arcade["ip"] != req.ip:
                     msg = f"Serial {req.serial} attempted allnet auth from bad IP {req.ip} (expected {arcade['ip']})."
-                    self.data.base.log_event(
+                    await self.data.base.log_event(
                         "allnet", "ALLNET_AUTH_BAD_IP", logging.ERROR, msg
                     )
                     self.logger.warning(msg)
 
                     resp.stat = ALLNET_STAT.bad_shop.value
                     resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-                    return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                    return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
                 
                 elif (not arcade["ip"] or arcade["ip"] is None) and self.config.server.strict_ip_checking:
                     msg = f"Serial {req.serial} attempted allnet auth from bad IP {req.ip}, but arcade {arcade['id']} has no IP set! (strict checking enabled)."
-                    self.data.base.log_event(
+                    await self.data.base.log_event(
                         "allnet", "ALLNET_AUTH_NO_SHOP_IP", logging.ERROR, msg
                     )
                     self.logger.warning(msg)
 
                     resp.stat = ALLNET_STAT.bad_shop.value
                     resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-                    return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                    return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
 
             country = (
@@ -238,34 +237,34 @@ class AllnetServlet:
         if req.game_id not in TitleServlet.title_registry:
             if not self.config.server.is_develop:
                 msg = f"Unrecognised game {req.game_id} attempted allnet auth from {request_ip}."
-                self.data.base.log_event(
+                await self.data.base.log_event(
                     "allnet", "ALLNET_AUTH_UNKNOWN_GAME", logging.WARN, msg
                 )
                 self.logger.warning(msg)
 
                 resp.stat = ALLNET_STAT.bad_game.value
                 resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-                return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
             else:
                 self.logger.info(
                     f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}"
                 )
-                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
+                resp.uri = f"http://{self.config.server.hostname}:{self.config.server.port}/{req.game_id}/{req.ver.replace('.', '')}/"
-                resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
+                resp.host = f"{self.config.server.hostname}:{self.config.server.port}"
                 
                 resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
                 resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
                 
                 self.logger.debug(f"Allnet response: {resp_str}")
-                return (resp_str + "\n").encode("utf-8")
+                return PlainTextResponse(resp_str + "\n")
 
         
         int_ver = req.ver.replace(".", "")
         resp.uri, resp.host = TitleServlet.title_registry[req.game_id].get_allnet_info(req.game_id, int(int_ver), req.serial)
 
         msg = f"{req.serial} authenticated from {request_ip}: {req.game_id} v{req.ver}"
-        self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
+        await self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
         self.logger.info(msg)
 
         resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
@@ -277,18 +276,19 @@ class AllnetServlet:
             request.responseHeaders.addRawHeader('Pragma', 'DFI')
             return self.to_dfi(resp_str)"""
 
-        return resp_str.encode("utf-8")
+        return PlainTextResponse(resp_str)
 
-    def handle_dlorder(self, request: Request, _: Dict):
+    async def handle_dlorder(self, request: Request):
         request_ip = Utils.get_ip_addr(request)
-        pragma_header = request.getHeader('Pragma')
+        pragma_header = request.headers.get('Pragma', "")
         is_dfi = pragma_header is not None and pragma_header == "DFI"
+        data = await request.body()
         
         try:
             if is_dfi:
-                req_urlencode = self.from_dfi(request.content.getvalue())
+                req_urlencode = self.from_dfi(data)
             else:
-                req_urlencode = request.content.getvalue().decode()
+                req_urlencode = data.decode()
 
             req_dict = self.allnet_req_to_dict(req_urlencode)
             if req_dict is None:
@@ -305,7 +305,7 @@ class AllnetServlet:
         except AllnetRequestException as e:
             if e.message != "":
                 self.logger.error(e)
-            return b""
+            return PlainTextResponse()
 
         self.logger.info(
             f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}"
@@ -316,54 +316,54 @@ class AllnetServlet:
             not self.config.allnet.allow_online_updates
             or not self.config.allnet.update_cfg_folder
         ):
-            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
+            return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n")
 
         else:  # TODO: Keychip check
             if path.exists(
                 f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
             ):
-                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
+                resp.uri = f"http://{self.config.server.hostname}:{self.config.server.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
 
             if path.exists(
                 f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
             ):
-                resp.uri += f"|http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
+                resp.uri += f"|http://{self.config.server.hostname}:{self.config.server.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
 
             self.logger.debug(f"Sending download uri {resp.uri}")
-            self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
+            await self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
 
             res_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
             """if is_dfi:
                 request.responseHeaders.addRawHeader('Pragma', 'DFI')
                 return self.to_dfi(res_str)"""
 
-            return res_str
+            return PlainTextResponse(res_str)
 
-    def handle_dlorder_ini(self, request: Request, match: Dict) -> bytes:
+    async def handle_dlorder_ini(self, request: Request) -> bytes:
-        if "file" not in match:
+        req_file = request.path_params.get("file", "").replace("%0A", "").replace("\n", "")
-            return b""
 
-        req_file = match["file"].replace("%0A", "")
+        if not req_file:
+            return PlainTextResponse(status_code=404)
 
         if path.exists(f"{self.config.allnet.update_cfg_folder}/{req_file}"):
             self.logger.info(f"Request for DL INI file {req_file} from {Utils.get_ip_addr(request)} successful")
-            self.data.base.log_event("allnet", "DLORDER_INI_SENT", logging.INFO, f"{Utils.get_ip_addr(request)} successfully recieved {req_file}")
+            await self.data.base.log_event("allnet", "DLORDER_INI_SENT", logging.INFO, f"{Utils.get_ip_addr(request)} successfully recieved {req_file}")
             
-            return open(
+            return PlainTextResponse(open(
-                f"{self.config.allnet.update_cfg_folder}/{req_file}", "rb"
+                f"{self.config.allnet.update_cfg_folder}/{req_file}", "r", encoding="utf-8"
-            ).read()
+            ).read())
 
         self.logger.info(f"DL INI File {req_file} not found")
-        return b""
+        return PlainTextResponse()
 
-    def handle_dlorder_report(self, request: Request, match: Dict) -> bytes:
+    async def handle_dlorder_report(self, request: Request) -> bytes:
-        req_raw = request.content.getvalue()
+        req_raw = await request.body()
         client_ip = Utils.get_ip_addr(request)
         try:
             req_dict: Dict = json.loads(req_raw)
         except Exception as e:
             self.logger.warning(f"Failed to parse DL Report: {e}")
-            return "NG"
+            return PlainTextResponse("NG")
         
         dl_data_type = DLIMG_TYPE.app
         dl_data = req_dict.get("appimage", {})
@@ -374,24 +374,24 @@ class AllnetServlet:
         
         if dl_data is None or not dl_data:
             self.logger.warning(f"Failed to parse DL Report: Invalid format - contains neither appimage nor optimage")
-            return "NG"
+            return PlainTextResponse("NG")
 
         rep = DLReport(dl_data, dl_data_type)
 
         if not rep.validate():
             self.logger.warning(f"Failed to parse DL Report: Invalid format - {rep.err}")
-            return "NG"
+            return PlainTextResponse("NG")
         
         msg = f"{rep.serial} @ {client_ip} reported {rep.rep_type.name} download state {rep.rf_state.name} for {rep.gd} v{rep.dav}:"\
               f" {rep.tdsc}/{rep.tsc} segments downloaded for working files {rep.wfl} with {rep.dfl if rep.dfl else 'none'} complete."
         
-        self.data.base.log_event("allnet", "DL_REPORT", logging.INFO, msg, dl_data)
+        await self.data.base.log_event("allnet", "DL_REPORT", logging.INFO, msg, dl_data)
         self.logger.info(msg)
 
-        return "OK"
+        return PlainTextResponse("OK")
 
-    def handle_loaderstaterecorder(self, request: Request, match: Dict) -> bytes:
+    async def handle_loaderstaterecorder(self, request: Request) -> bytes:
-        req_data = request.content.getvalue()
+        req_data = await request.body()
         sections = req_data.decode("utf-8").split("\r\n")
         
         req_dict = dict(urllib.parse.parse_qsl(sections[0]))
@@ -403,130 +403,17 @@ class AllnetServlet:
         ip = Utils.get_ip_addr(request)
 
         if serial is None or num_files_dld is None or num_files_to_dl is None or dl_state is None:
-            return "NG".encode()
+            return PlainTextResponse("NG")
 
         self.logger.info(f"LoaderStateRecorder Request from {ip} {serial}: {num_files_dld}/{num_files_to_dl} Files download (State: {dl_state})")
-        return "OK".encode()
+        return PlainTextResponse("OK")
     
-    def handle_alive(self, request: Request, match: Dict) -> bytes:
+    async def handle_alive(self, request: Request) -> bytes:
-        return "OK".encode()
+        return PlainTextResponse("OK")
 
-    def handle_billing_request(self, request: Request, _: Dict):
+    async def handle_naomitest(self, request: Request) -> bytes:
-        req_raw = request.content.getvalue()
-        
-        if request.getHeader('Content-Type') == "application/octet-stream":
-            req_unzip = zlib.decompressobj(-zlib.MAX_WBITS).decompress(req_raw)
-        else:
-            req_unzip = req_raw
-        
-        req_dict = self.billing_req_to_dict(req_unzip)
-        request_ip = Utils.get_ip_addr(request)
-        
-        if req_dict is None:
-            self.logger.error(f"Failed to parse request {request.content.getvalue()}")
-            return b""
-
-        self.logger.debug(f"request {req_dict}")
-
-        rsa = RSA.import_key(open(self.config.billing.signing_key, "rb").read())
-        signer = PKCS1_v1_5.new(rsa)
-        digest = SHA.new()
-        traces: List[TraceData] = []
-        try:
-            req = BillingInfo(req_dict[0])
-        except KeyError as e:
-            self.logger.error(f"Billing request failed to parse: {e}")
-            return f"result=5&linelimit=&message=field is missing or formatting is incorrect\r\n".encode()
-
-        for x in range(1, len(req_dict)):
-            if not req_dict[x]:
-                continue
-            
-            try:
-                tmp = TraceData(req_dict[x])
-                if tmp.trace_type == TraceDataType.CHARGE:
-                    tmp = TraceDataCharge(req_dict[x])
-                elif tmp.trace_type == TraceDataType.EVENT:
-                    tmp = TraceDataEvent(req_dict[x])
-                elif tmp.trace_type == TraceDataType.CREDIT:
-                    tmp = TraceDataCredit(req_dict[x])
-                
-                traces.append(tmp)
-            
-            except KeyError as e:
-                self.logger.warn(f"Tracelog failed to parse: {e}")
-
-        kc_serial_bytes = req.keychipid.encode()
-        
-
-        machine = self.data.arcade.get_machine(req.keychipid)
-        if machine is None and not self.config.server.allow_unregistered_serials:
-            msg = f"Unrecognised serial {req.keychipid} attempted billing checkin from {request_ip} for {req.gameid} v{req.gamever}."
-            self.data.base.log_event(
-                "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg
-            )
-            self.logger.warning(msg)
-
-            return f"result=1&requestno={req.requestno}&message=Keychip Serial bad\r\n".encode()
-
-        msg = (
-            f"Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount "
-            f"{req.playcnt} billing_type {req.billingtype.name} nearfull {req.nearfull} playlimit {req.playlimit}"
-        )
-        self.logger.info(msg)
-        self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, msg)
-        if req.traceleft > 0:
-            self.logger.warn(f"{req.traceleft} unsent tracelogs")
-        kc_playlimit = req.playlimit
-        kc_nearfull = req.nearfull
-
-        while req.playcnt > req.playlimit:
-            kc_playlimit += 1024
-            kc_nearfull += 1024
-
-        playlimit = kc_playlimit
-        nearfull = kc_nearfull + (req.billingtype.value * 0x00010000)
-
-        digest.update(playlimit.to_bytes(4, "little") + kc_serial_bytes)
-        playlimit_sig = signer.sign(digest).hex()
-
-        digest = SHA.new()
-        digest.update(nearfull.to_bytes(4, "little") + kc_serial_bytes)
-        nearfull_sig = signer.sign(digest).hex()
-
-        # TODO: playhistory
-
-        #resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig)
-        resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig, req.requestno, req.protocolver)
-
-        resp_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\r\n"
-
-        self.logger.debug(f"response {vars(resp)}")
-        if req.traceleft > 0:
-            self.logger.info(f"Requesting 20 more of {req.traceleft} unsent tracelogs")
-            return f"result=6&waittime=0&linelimit=20\r\n".encode()
-
-        return resp_str.encode("utf-8")
-
-    def handle_naomitest(self, request: Request, _: Dict) -> bytes:
         self.logger.info(f"Ping from {Utils.get_ip_addr(request)}")
-        return b"naomi ok"
+        return PlainTextResponse("naomi ok")
-
-    def billing_req_to_dict(self, data: bytes):
-        """
-        Parses an billing request string into a python dictionary
-        """
-        try:
-            sections = data.decode("ascii").split("\r\n")
-
-            ret = []
-            for x in sections:
-                ret.append(dict(urllib.parse.parse_qsl(x)))
-            return ret
-
-        except Exception as e:
-            self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
-            return None
 
     def allnet_req_to_dict(self, data: str) -> Optional[List[Dict[str, Any]]]:
         """
@@ -554,6 +441,147 @@ class AllnetServlet:
         zipped = zlib.compress(unzipped)
         return base64.b64encode(zipped)
 
+class BillingServlet:
+    def __init__(self, core_cfg: CoreConfig, cfg_folder: str) -> None:
+        self.config = core_cfg
+        self.config_folder = cfg_folder
+        self.data = Data(core_cfg)
+
+        self.logger = logging.getLogger("billing")
+        if not hasattr(self.logger, "initialized"):
+            log_fmt_str = "[%(asctime)s] Billing | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "billing"),
+                when="d",
+                backupCount=10,
+            )
+            fileHandler.setFormatter(log_fmt)
+
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
+
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(core_cfg.allnet.loglevel)
+            coloredlogs.install(
+                level=core_cfg.billing.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.initialized = True
+    
+    def billing_req_to_dict(self, data: bytes):
+        """
+        Parses an billing request string into a python dictionary
+        """
+        try:
+            sections = data.decode("ascii").split("\r\n")
+
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
+
+        except Exception as e:
+            self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
+            return None
+    
+    async def handle_billing_request(self, request: Request):
+        req_raw = await request.body()
+        
+        if request.headers.get('Content-Type', '') == "application/octet-stream":
+            req_unzip = zlib.decompressobj(-zlib.MAX_WBITS).decompress(req_raw)
+        else:
+            req_unzip = req_raw
+        
+        req_dict = self.billing_req_to_dict(req_unzip)
+        request_ip = Utils.get_ip_addr(request)
+        
+        if req_dict is None:
+            self.logger.error(f"Failed to parse request {req_raw}")
+            return PlainTextResponse()
+
+        self.logger.debug(f"request {req_dict}")
+
+        rsa = RSA.import_key(open(self.config.billing.signing_key, "rb").read())
+        signer = PKCS1_v1_5.new(rsa)
+        digest = SHA.new()
+        traces: List[TraceData] = []
+        try:
+            req = BillingInfo(req_dict[0])
+        except KeyError as e:
+            self.logger.error(f"Billing request failed to parse: {e}")
+            return PlainTextResponse("result=5&linelimit=&message=field is missing or formatting is incorrect\r\n")
+
+        for x in range(1, len(req_dict)):
+            if not req_dict[x]:
+                continue
+            
+            try:
+                tmp = TraceData(req_dict[x])
+                if tmp.trace_type == TraceDataType.CHARGE:
+                    tmp = TraceDataCharge(req_dict[x])
+                elif tmp.trace_type == TraceDataType.EVENT:
+                    tmp = TraceDataEvent(req_dict[x])
+                elif tmp.trace_type == TraceDataType.CREDIT:
+                    tmp = TraceDataCredit(req_dict[x])
+                
+                traces.append(tmp)
+            
+            except KeyError as e:
+                self.logger.warn(f"Tracelog failed to parse: {e}")
+
+        kc_serial_bytes = req.keychipid.encode()
+        
+
+        machine = await self.data.arcade.get_machine(req.keychipid)
+        if machine is None and not self.config.server.allow_unregistered_serials:
+            msg = f"Unrecognised serial {req.keychipid} attempted billing checkin from {request_ip} for {req.gameid} v{req.gamever}."
+            await self.data.base.log_event(
+                "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg
+            )
+            self.logger.warning(msg)
+
+            return PlainTextResponse(f"result=1&requestno={req.requestno}&message=Keychip Serial bad\r\n")
+
+        msg = (
+            f"Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount "
+            f"{req.playcnt} billing_type {req.billingtype.name} nearfull {req.nearfull} playlimit {req.playlimit}"
+        )
+        self.logger.info(msg)
+        await self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, msg)
+        if req.traceleft > 0:
+            self.logger.warn(f"{req.traceleft} unsent tracelogs")
+        kc_playlimit = req.playlimit
+        kc_nearfull = req.nearfull
+
+        while req.playcnt > req.playlimit:
+            kc_playlimit += 1024
+            kc_nearfull += 1024
+
+        playlimit = kc_playlimit
+        nearfull = kc_nearfull + (req.billingtype.value * 0x00010000)
+
+        digest.update(playlimit.to_bytes(4, "little") + kc_serial_bytes)
+        playlimit_sig = signer.sign(digest).hex()
+
+        digest = SHA.new()
+        digest.update(nearfull.to_bytes(4, "little") + kc_serial_bytes)
+        nearfull_sig = signer.sign(digest).hex()
+
+        # TODO: playhistory
+
+        resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig, req.requestno, req.protocolver)
+
+        resp_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\r\n"
+
+        self.logger.debug(f"response {vars(resp)}")
+        if req.traceleft > 0:
+            self.logger.info(f"Requesting 20 more of {req.traceleft} unsent tracelogs")
+            return PlainTextResponse("result=6&waittime=0&linelimit=20\r\n")
+        
+        return PlainTextResponse(resp_str)
 
 class AllnetPowerOnRequest:
     def __init__(self, req: Dict) -> None:
@@ -613,7 +641,6 @@ class AllnetPowerOnResponse3(AllnetPowerOnResponse):
         self.minute = None
         self.second = None
 
-
 class AllnetPowerOnResponse2(AllnetPowerOnResponse):
     def __init__(self) -> None:
         super().__init__()
@@ -623,7 +650,6 @@ class AllnetPowerOnResponse2(AllnetPowerOnResponse):
         self.timezone = "+09:00"
         self.res_class = "PowerOnResponseV2"
 
-
 class AllnetDownloadOrderRequest:
     def __init__(self, req: Dict) -> None:
         self.game_id = req.get("game_id", "")
@@ -631,7 +657,6 @@ class AllnetDownloadOrderRequest:
         self.serial = req.get("serial", "")
         self.encode = req.get("encode", "")
 
-
 class AllnetDownloadOrderResponse:
     def __init__(self, stat: int = 1, serial: str = "", uri: str = "") -> None:
         self.stat = stat
@@ -669,7 +694,7 @@ class BillingInfo:
             self.boardid = str(data.get("boardid", None))
             self.tenpoip = str(data.get("tenpoip", None))
             self.libalibver = float(data.get("libalibver", None))
-            self.datamax = int(data.get("datamax", None))
+            self.data.max = int(data.get("datamax", None))
             self.billingtype = BillingType(int(data.get("billingtype", None)))
             self.protocolver = float(data.get("protocolver", None))
             self.operatingfix = bool(data.get("operatingfix", None))
@@ -781,7 +806,6 @@ class BillingResponse:
         # playhistory -> YYYYMM/C:...
         # YYYY -> 4 digit year, MM -> 2 digit month, C -> Playcount during that period
 
-
 class AllnetRequestException(Exception):
     def __init__(self, message="") -> None:
         self.message = message
@@ -849,3 +873,46 @@ class DLReport:
             return False
         
         return True
+
+cfg_dir = environ.get("DIANA_CFG_DIR", "config")
+cfg: CoreConfig = CoreConfig()
+if path.exists(f"{cfg_dir}/core.yaml"):
+    cfg.update(yaml.safe_load(open(f"{cfg_dir}/core.yaml")))
+
+if not path.exists(cfg.server.log_dir):
+    mkdir(cfg.server.log_dir)
+
+if not access(cfg.server.log_dir, W_OK):
+    print(
+        f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+    )
+    exit(1)
+
+billing = BillingServlet(cfg, cfg_dir)
+app_billing = Starlette(
+    cfg.server.is_develop, 
+    [        
+        Route("/request", billing.handle_billing_request, methods=["POST"]),
+        Route("/request/", billing.handle_billing_request, methods=["POST"]),
+    ]
+)
+
+allnet = AllnetServlet(cfg, cfg_dir)
+route_lst = [
+    Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]),
+    Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]),
+    Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]),
+    Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]),
+    Route("/naomitest.html", allnet.handle_naomitest),
+]
+
+if cfg.allnet.allow_online_updates:
+    route_lst += [
+        Route("/report-api/Report", allnet.handle_dlorder_report, methods=["POST"]),
+        Route("/dl/ini/{file:str}", allnet.handle_dlorder_ini),
+    ]
+
+app_allnet = Starlette(
+    cfg.server.is_develop, 
+    route_lst
+)

core/app.py

@@ -0,0 +1,94 @@
+import yaml
+import logging
+import coloredlogs
+from logging.handlers import TimedRotatingFileHandler
+from starlette.routing import Route
+from starlette.requests import Request
+from starlette.applications import Starlette
+from starlette.responses import PlainTextResponse
+from os import environ, path, mkdir, W_OK, access
+from typing import List
+
+from core import CoreConfig, TitleServlet, MuchaServlet, AllnetServlet, BillingServlet, AimedbServlette
+from core.frontend import FrontendServlet
+
+async def dummy_rt(request: Request):
+    return PlainTextResponse("Service OK")
+
+cfg_dir = environ.get("ARTEMIS_CFG_DIR", "config")
+cfg: CoreConfig = CoreConfig()
+if path.exists(f"{cfg_dir}/core.yaml"):
+    cfg.update(yaml.safe_load(open(f"{cfg_dir}/core.yaml")))
+
+if not path.exists(cfg.server.log_dir):
+    mkdir(cfg.server.log_dir)
+
+if not access(cfg.server.log_dir, W_OK):
+    print(
+        f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+    )
+    exit(1)
+
+logger = logging.getLogger("core")
+log_fmt_str = "[%(asctime)s] Core | %(levelname)s | %(message)s"
+log_fmt = logging.Formatter(log_fmt_str)
+
+fileHandler = TimedRotatingFileHandler(
+    "{0}/{1}.log".format(cfg.server.log_dir, "core"), when="d", backupCount=10
+)
+fileHandler.setFormatter(log_fmt)
+
+consoleHandler = logging.StreamHandler()
+consoleHandler.setFormatter(log_fmt)
+
+logger.addHandler(fileHandler)
+logger.addHandler(consoleHandler)
+
+log_lv = logging.DEBUG if cfg.server.is_develop else logging.INFO
+logger.setLevel(log_lv)
+coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
+
+logger.info(f"Artemis starting in {'develop' if cfg.server.is_develop else 'production'} mode")
+
+title = TitleServlet(cfg, cfg_dir) # This has to be loaded first to load plugins
+mucha = MuchaServlet(cfg, cfg_dir)
+
+route_lst: List[Route] = [
+    # Allnet
+    
+    # Mucha
+    Route("/mucha_front/boardauth.do", mucha.handle_boardauth, methods=["POST"]),
+    Route("/mucha_front/updatacheck.do", mucha.handle_updatecheck, methods=["POST"]),
+    Route("/mucha_front/downloadstate.do", mucha.handle_dlstate, methods=["POST"]),
+    # General
+    Route("/", dummy_rt),
+    Route("/robots.txt", FrontendServlet.robots)
+]
+
+if not cfg.billing.standalone:
+    billing = BillingServlet(cfg, cfg_dir)
+    route_lst += [
+        Route("/request", billing.handle_billing_request, methods=["POST"]),
+        Route("/request/", billing.handle_billing_request, methods=["POST"]),
+    ]
+
+if not cfg.allnet.standalone:
+    allnet = AllnetServlet(cfg, cfg_dir)
+    route_lst += [
+        Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]),
+        Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]),
+        Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]),
+        Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]),
+        Route("/naomitest.html", allnet.handle_naomitest),
+    ]
+    
+    if cfg.allnet.allow_online_updates:
+        route_lst += [
+            Route("/report-api/Report", allnet.handle_dlorder_report, methods=["POST"]),
+            Route("/dl/ini/{file:str}", allnet.handle_dlorder_ini),
+        ]
+
+for code, game in title.title_registry.items():
+    route_lst += game.get_routes()
+
+app = Starlette(cfg.server.is_develop, route_lst)

core/config.py

@@ -1,17 +1,49 @@
 import logging, os
 from typing import Any
 
-
 class ServerConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
     def listen_address(self) -> str:
+        """
+        Address Artemis will bind to and listen on
+        """
         return CoreConfig.get_config_field(
             self.__config, "core", "server", "listen_address", default="127.0.0.1"
         )
     
+    @property
+    def hostname(self) -> str:
+        """
+        Hostname sent to games
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "hostname", default="localhost"
+        )
+    
+    @property
+    def port(self) -> int:
+        """
+        Port the game will listen on
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "port", default=80
+        )
+
+    @property
+    def ssl_key(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "ssl_key", default="cert/title.key"
+        )
+
+    @property
+    def ssl_cert(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "ssl_cert", default="cert/title.pem"
+        )
+
     @property
     def allow_user_registration(self) -> bool:
         return CoreConfig.get_config_field(
@@ -43,9 +75,23 @@ class ServerConfig:
         )
 
     @property
-    def threading(self) -> bool:
+    def proxy_port(self) -> int:
+        """
+        What port the proxy is listening on. This will be sent instead of 'port' if 
+        is_using_proxy is True and this value is non-zero
+        """
         return CoreConfig.get_config_field(
-            self.__config, "core", "server", "threading", default=False
+            self.__config, "core", "title", "proxy_port", default=0
+        )
+
+    @property
+    def proxy_port_ssl(self) -> int:
+        """
+        What port the proxy is listening for secure connections on. This will be sent 
+        instead of 'port' if is_using_proxy is True and this value is non-zero
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "proxy_port_ssl", default=0
         )
 
     @property
@@ -66,7 +112,6 @@ class ServerConfig:
             self.__config, "core", "server", "strict_ip_checking", default=False
         )
 
-
 class TitleConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
@@ -79,36 +124,6 @@ class TitleConfig:
             )
         )
 
-    @property
-    def hostname(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "hostname", default="localhost"
-        )
-
-    @property
-    def port(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "port", default=8080
-        )
-        
-    @property
-    def port_ssl(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "port_ssl", default=0
-        )
-
-    @property
-    def ssl_key(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "ssl_key", default="cert/title.key"
-        )
-
-    @property
-    def ssl_cert(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "ssl_cert", default="cert/title.pem"
-        )
-
     @property
     def reboot_start_time(self) -> str:
         return CoreConfig.get_config_field(
@@ -121,7 +136,6 @@ class TitleConfig:
             self.__config, "core", "title", "reboot_end_time", default=""
         )
 
-
 class DatabaseConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
@@ -176,16 +190,6 @@ class DatabaseConfig:
             )
         )
 
-    @property
-    def user_table_autoincrement_start(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config,
-            "core",
-            "database",
-            "user_table_autoincrement_start",
-            default=10000,
-        )
-
     @property
     def enable_memcached(self) -> bool:
         return CoreConfig.get_config_field(
@@ -198,13 +202,12 @@ class DatabaseConfig:
             self.__config, "core", "database", "memcached_host", default="localhost"
         )
 
-
 class FrontendConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
-    def enable(self) -> int:
+    def enable(self) -> bool:
         return CoreConfig.get_config_field(
             self.__config, "core", "frontend", "enable", default=False
         )
@@ -212,7 +215,7 @@ class FrontendConfig:
     @property
     def port(self) -> int:
         return CoreConfig.get_config_field(
-            self.__config, "core", "frontend", "port", default=8090
+            self.__config, "core", "frontend", "port", default=8080
         )
 
     @property
@@ -223,17 +226,20 @@ class FrontendConfig:
             )
         )
     
+    @property
+    def secret(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "frontend", "secret", default=""
+        )
 
 class AllnetConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
-    def loglevel(self) -> int:
+    def standalone(self) -> bool:
-        return CoreConfig.str_to_loglevel(
+        return CoreConfig.get_config_field(
-            CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "standalone", default=False
-                self.__config, "core", "allnet", "loglevel", default="info"
-            )
         )
     
     @property
@@ -243,9 +249,11 @@ class AllnetConfig:
         )
 
     @property
-    def ip_check(self) -> bool:
+    def loglevel(self) -> int:
-        return CoreConfig.get_config_field(
+        return CoreConfig.str_to_loglevel(
-            self.__config, "core", "allnet", "ip_check", default=False
+            CoreConfig.get_config_field(
+                self.__config, "core", "allnet", "loglevel", default="info"
+            )
         )
 
     @property
@@ -260,11 +268,24 @@ class AllnetConfig:
             self.__config, "core", "allnet", "update_cfg_folder", default=""
         )
 
-
 class BillingConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
     
+    @property
+    def standalone(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "standalone", default=True
+        )
+
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "billing", "loglevel", default="info"
+            )
+        )
+
     @property
     def port(self) -> int:
         return CoreConfig.get_config_field(
@@ -289,11 +310,16 @@ class BillingConfig:
             self.__config, "core", "billing", "signing_key", default="cert/billing.key"
         )
 
-
 class AimedbConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
+    @property
+    def enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "enable", default=True
+        )
+
     @property
     def loglevel(self) -> int:
         return CoreConfig.str_to_loglevel(
@@ -326,17 +352,10 @@ class AimedbConfig:
             self.__config, "core", "aimedb", "id_lifetime_seconds", default=86400
         )
 
-
 class MuchaConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
-    @property
-    def enable(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "mucha", "enable", default=False
-        )
-
     @property
     def loglevel(self) -> int:
         return CoreConfig.str_to_loglevel(
@@ -345,13 +364,6 @@ class MuchaConfig:
             )
         )
 
-    @property
-    def hostname(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "mucha", "hostname", default="localhost"
-        )
-
-
 class CoreConfig(dict):
     def __init__(self) -> None:
         self.server = ServerConfig(self)
@@ -374,6 +386,19 @@ class CoreConfig(dict):
         else:
             return logging.INFO
     
+    @classmethod
+    def loglevel_to_str(cls, level: int) -> str:
+        if level == logging.ERROR:
+            return "error"
+        elif level == logging.WARN:
+            return "warn"
+        elif level == logging.INFO:
+            return "info"
+        elif level == logging.DEBUG:
+            return "debug"
+        else:
+            return "notset"
+
     @classmethod
     def get_config_field(
         cls, __config: dict, module, *path: str, default: Any = ""

core/data/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

core/data/alembic/alembic.ini

@@ -0,0 +1,64 @@
+# A generic, single database configuration.
+
+[alembic]
+script_location=.
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to migrations//versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat migrations//versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

core/data/alembic/env.py

@@ -0,0 +1,81 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+from core.data.schema.base import metadata
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    raise Exception('Not implemented or configured!')
+
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    ini_section = config.get_section(config.config_ini_section)
+    overrides = context.get_x_argument(as_dictionary=True)
+    for override in overrides:
+        ini_section[override] = overrides[override]
+
+    connectable = engine_from_config(
+        ini_section,
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=True,
+            compare_server_default=True,
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

core/data/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

core/data/alembic/versions/835b862f9bf0_initial_migration.py

@@ -0,0 +1,24 @@
+"""Initial Migration
+
+Revision ID: 835b862f9bf0
+Revises: 
+Create Date: 2024-01-09 13:06:10.787432
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '835b862f9bf0'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

core/data/alembic/versions/d8950c7ce2fc_remove_old_db_mgmt_system.py

@@ -0,0 +1,29 @@
+"""Remove old db mgmt system
+
+Revision ID: d8950c7ce2fc
+Revises: 835b862f9bf0
+Create Date: 2024-01-09 13:43:51.381175
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd8950c7ce2fc'
+down_revision = '835b862f9bf0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_table("schema_versions")
+
+
+def downgrade():
+    op.create_table(
+        "schema_versions",
+        sa.Column("game", sa.String(4), primary_key=True, nullable=False),
+        sa.Column("version", sa.Integer, nullable=False, server_default="1"),
+        mysql_charset="utf8mb4",
+    )

core/data/database.py

@@ -1,13 +1,14 @@
 import logging, coloredlogs
-from typing import Optional, Dict, List
+from typing import Optional
 from sqlalchemy.orm import scoped_session, sessionmaker
-from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
 from logging.handlers import TimedRotatingFileHandler
-import importlib, os
+import os
 import secrets, string
 import bcrypt
 from hashlib import sha256
+import alembic.config
+import glob
 
 from core.config import CoreConfig
 from core.data.schema import *
@@ -15,7 +16,6 @@ from core.utils import Utils
 
 
 class Data:
-    current_schema_version = 6
     engine = None
     session = None
     user = None
@@ -77,281 +77,144 @@ class Data:
             )
             self.logger.handler_set = True  # type: ignore
 
+    def __alembic_cmd(self, command: str, *args: str) -> None:
+        old_dir = os.path.abspath(os.path.curdir)
+        base_dir = os.path.join(os.path.abspath(os.path.curdir), 'core', 'data', 'alembic')
+        alembicArgs = [
+            "-c",
+            os.path.join(base_dir, "alembic.ini"),
+            "-x",
+            f"script_location={base_dir}",
+            "-x",
+            f"sqlalchemy.url={self.__url}",
+            command,
+        ]
+        alembicArgs.extend(args)
+        os.chdir(base_dir)
+        alembic.config.main(argv=alembicArgs)
+        os.chdir(old_dir)
+
     def create_database(self):
         self.logger.info("Creating databases...")
-        try:
+        metadata.create_all(
-            metadata.create_all(self.__engine.connect())
+            self.engine,
-        except SQLAlchemyError as e:
+            checkfirst=True,
-            self.logger.error(f"Failed to create databases! {e}")
-            return
-
-        games = Utils.get_all_titles()
-        for game_dir, game_mod in games.items():
-            try:
-                if hasattr(game_mod, "database") and hasattr(
-                    game_mod, "current_schema_version"
-                ):
-                    game_mod.database(self.config)
-                metadata.create_all(self.__engine.connect())
-
-                self.base.touch_schema_ver(
-                    game_mod.current_schema_version, game_mod.game_codes[0]
         )
 
-            except Exception as e:
+        for _, mod in Utils.get_all_titles().items():
-                self.logger.warning(
-                    f"Could not load database schema from {game_dir} - {e}"
-                )
-
-        self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
-        self.base.set_schema_ver(self.current_schema_version)
-
-        self.logger.info(
-            f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
-        )
-        self.user.reset_autoincrement(
-            self.config.database.user_table_autoincrement_start
-        )
-
-    def recreate_database(self):
-        self.logger.info("Dropping all databases...")
-        self.base.execute("SET FOREIGN_KEY_CHECKS=0")
-        try:
-            metadata.drop_all(self.__engine.connect())
-        except SQLAlchemyError as e:
-            self.logger.error(f"Failed to drop databases! {e}")
-            return
-
-        for root, dirs, files in os.walk("./titles"):
-            for dir in dirs:
-                if not dir.startswith("__"):
-                    try:
-                        mod = importlib.import_module(f"titles.{dir}")
-
-                        try:
             if hasattr(mod, "database"):
                 mod.database(self.config)
-                            metadata.drop_all(self.__engine.connect())
+                metadata.create_all(
-
+                    self.engine,
-                        except Exception as e:
+                    checkfirst=True,
-                            self.logger.warning(
-                                f"Could not load database schema from {dir} - {e}"
                 )
 
-                    except ImportError as e:
+        # Stamp the end revision as if alembic had created it, so it can take off after this.
-                        self.logger.warning(
+        self.__alembic_cmd(
-                            f"Failed to load database schema dir {dir} - {e}"
+            "stamp",
-                        )
+            "head",
-            break
-
-        self.base.execute("SET FOREIGN_KEY_CHECKS=1")
-
-        self.create_database()
-
-    def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
-        old_ver = self.base.get_schema_ver(game)
-        sql = ""
-        if version is None:
-            if not game == "CORE":
-                titles = Utils.get_all_titles()
-
-                for folder, mod in titles.items():
-                    if not mod.game_codes[0] == game:
-                        continue
-
-                    if hasattr(mod, "current_schema_version"):
-                        version = mod.current_schema_version
-
-                    else:
-                        self.logger.warning(
-                            f"current_schema_version not found for {folder}"
         )
 
-            else:
+    def schema_upgrade(self, ver: str = None):
-                version = self.current_schema_version
+        self.__alembic_cmd(
-
+            "upgrade",
-        if version is None:
+            "head",
-            self.logger.warning(
-                f"Could not determine latest version for {game}, please specify --version"
         )
 
-        if old_ver is None:
+    async def create_owner(self, email: Optional[str] = None, code: Optional[str] = "00000000000000000000") -> None:
-            self.logger.error(
-                f"Schema for game {game} does not exist, did you run the creation script?"
-            )
-            return
-
-        if old_ver == version:
-            self.logger.info(
-                f"Schema for game {game} is already version {old_ver}, nothing to do"
-            )
-            return
-
-        if action == "upgrade":
-            for x in range(old_ver, version):
-                if not os.path.exists(
-                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql"
-                ):
-                    self.logger.error(
-                        f"Could not find {action} script {game.upper()}_{x + 1}_{action}.sql in core/data/schema/versions folder"
-                    )
-                    return
-
-                with open(
-                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql",
-                    "r",
-                    encoding="utf-8",
-                ) as f:
-                    sql = f.read()
-
-                result = self.base.execute(sql)
-                if result is None:
-                    self.logger.error("Error execuing sql script!")
-                    return None
-
-        else:
-            for x in range(old_ver, version, -1):
-                if not os.path.exists(
-                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql"
-                ):
-                    self.logger.error(
-                        f"Could not find {action} script {game.upper()}_{x - 1}_{action}.sql in core/data/schema/versions folder"
-                    )
-                    return
-
-                with open(
-                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql",
-                    "r",
-                    encoding="utf-8",
-                ) as f:
-                    sql = f.read()
-
-                result = self.base.execute(sql)
-                if result is None:
-                    self.logger.error("Error execuing sql script!")
-                    return None
-
-        result = self.base.set_schema_ver(version, game)
-        if result is None:
-            self.logger.error("Error setting version in schema_version table!")
-            return None
-
-        self.logger.info(f"Successfully migrated {game} to schema version {version}")
-
-    def create_owner(self, email: Optional[str] = None) -> None:
         pw = "".join(
             secrets.choice(string.ascii_letters + string.digits) for i in range(20)
         )
         hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt())
 
-        user_id = self.user.create_user(email=email, permission=255, password=hash)
+        user_id = await self.user.create_user("sysowner", email, hash.decode(), 255)
         if user_id is None:
             self.logger.error(f"Failed to create owner with email {email}")
             return
 
-        card_id = self.card.create_card(user_id, "00000000000000000000")
+        card_id = await self.card.create_card(user_id, code)
         if card_id is None:
             self.logger.error(f"Failed to create card for owner with id {user_id}")
             return
 
         self.logger.warning(
-            f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!"
+            f"Successfully created owner with email {email}, access code {code}, and password {pw} Make sure to change this password and assign a real card ASAP!"
         )
     
-    def migrate_card(self, old_ac: str, new_ac: str, should_force: bool) -> None:
+    async def migrate(self) -> None:
-        if old_ac == new_ac:
+        exist = await self.base.execute("SELECT * FROM alembic_version")
-            self.logger.error("Both access codes are the same!")
+        if exist is not None:
+            self.logger.warn("No need to migrate as you have already migrated to alembic. If you are trying to upgrade the schema, use `upgrade` instead!")
             return
         
-        new_card = self.card.get_card_by_access_code(new_ac)
+        self.logger.info("Upgrading to latest with legacy system")
-        if new_card is None:
+        if not await self.legacy_upgrade():
-            self.card.update_access_code(old_ac, new_ac)
+            self.logger.warn("No need to migrate as you have already deleted the old schema_versions system. If you are trying to upgrade the schema, use `upgrade` instead!")
             return
+        self.logger.info("Done")
         
-        if not should_force:
+        self.logger.info("Stamp with initial revision")
-            self.logger.warning(
+        self.__alembic_cmd(
-                f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
+            "stamp",
-                f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
+            "835b862f9bf0",
         )
-            return
 
-        self.logger.info(
+        self.logger.info("Upgrade")
-            f"All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
+        self.__alembic_cmd(
+            "upgrade",
+            "head",
         )
-        self.card.delete_card(new_card["id"])
-        self.card.update_access_code(old_ac, new_ac)
     
-        hanging_user = self.user.get_user(new_card["user"])
+    async def legacy_upgrade(self) -> bool:
-        if hanging_user["password"] is None:
+        vers = await self.base.execute("SELECT * FROM schema_versions")
-            self.logger.info(f"Delete hanging user {hanging_user['id']}")
+        if vers is None:
-            self.user.delete_user(hanging_user["id"])
+            self.logger.warn("Cannot legacy upgrade, schema_versions table unavailable!")
+            return False
         
-    def delete_hanging_users(self) -> None:
+        db_vers = {}
-        """
+        for x in vers:
-        Finds and deletes users that have not registered for the webui that have no cards assocated with them.
+            db_vers[x['game']] = x['version']
-        """
-        unreg_users = self.user.get_unregistered_users()
-        if unreg_users is None:
-            self.logger.error("Error occoured finding unregistered users")
         
-        for user in unreg_users:
+        core_now_ver = int(db_vers['CORE']) + 1
-            cards = self.card.get_user_cards(user["id"])
+        while os.path.exists(f"core/data/schema/versions/CORE_{core_now_ver}_upgrade.sql"):
-            if cards is None:
+            with open(f"core/data/schema/versions/CORE_{core_now_ver}_upgrade.sql", "r") as f:
-                self.logger.error(f"Error getting cards for user {user['id']}")
+                result = await self.base.execute(f.read())
-                continue
                 
-            if not cards:
-                self.logger.info(f"Delete hanging user {user['id']}")
-                self.user.delete_user(user["id"])
-
-    def autoupgrade(self) -> None:
-        all_game_versions = self.base.get_all_schema_vers()
-        if all_game_versions is None:
-            self.logger.warning("Failed to get schema versions")
-            return
-
-        all_games = Utils.get_all_titles()
-        all_games_list: Dict[str, int] = {}
-        for _, mod in all_games.items():
-            if hasattr(mod, "current_schema_version"):
-                all_games_list[mod.game_codes[0]] = mod.current_schema_version
-
-        for x in all_game_versions:
-            failed = False
-            game = x["game"].upper()
-            update_ver = int(x["version"])
-            latest_ver = all_games_list.get(game, 1)
-            if game == "CORE":
-                latest_ver = self.current_schema_version
-
-            if update_ver == latest_ver:
-                self.logger.info(f"{game} is already latest version")
-                continue
-
-            for y in range(update_ver + 1, latest_ver + 1):
-                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
-                    with open(
-                        f"core/data/schema/versions/{game}_{y}_upgrade.sql",
-                        "r",
-                        encoding="utf-8",
-                    ) as f:
-                        sql = f.read()
-
-                    result = self.base.execute(sql)
                 if result is None:
-                        self.logger.error(
+                    self.logger.error(f"Invalid upgrade script CORE_{core_now_ver}_upgrade.sql")
-                            f"Error execuing sql script for game {game} v{y}!"
-                        )
-                        failed = True
                     break
-                else:
-                    self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
-                    failed = True
 
-            if not failed:
+                result = await self.base.execute(f"UPDATE schema_versions SET version = {core_now_ver} WHERE game = 'CORE'")
-                self.base.set_schema_ver(latest_ver, game)
+                if result is None:
+                    self.logger.error(f"Failed to update schema version for CORE to {core_now_ver}")
+                    break
+            
+            self.logger.info(f"Upgrade CORE to version {core_now_ver}")
+            core_now_ver += 1
+        
+        for _, mod in Utils.get_all_titles().items():
+            game_codes = getattr(mod, "game_codes", [])
+            for game in game_codes:
+                if game not in db_vers:
+                    self.logger.warn(f"{game} does not have an antry in schema_versions, skipping")
+                    continue
+
+                now_ver = int(db_vers[game]) + 1
+                while os.path.exists(f"core/data/schema/versions/{game}_{now_ver}_upgrade.sql"):
+                    with open(f"core/data/schema/versions/{game}_{now_ver}_upgrade.sql", "r") as f:
+                        result = await self.base.execute(f.read())
+                        
+                        if result is None:
+                            self.logger.error(f"Invalid upgrade script {game}_{now_ver}_upgrade.sql")
+                            break
+
+                        result = await self.base.execute(f"UPDATE schema_versions SET version = {now_ver} WHERE game = '{game}'")
+                        if result is None:
+                            self.logger.error(f"Failed to update schema version for {game} to {now_ver}")
+                            break
+
+                    self.logger.info(f"Upgrade {game} to version {now_ver}")
+                    now_ver += 1
+        
+        return True
 
-    def show_versions(self) -> None:
-        all_game_versions = self.base.get_all_schema_vers()
-        for ver in all_game_versions:
-            self.logger.info(f"{ver['game']} -> v{ver['version']}")

core/data/schema/arcade.py

@@ -69,7 +69,7 @@ arcade_owner = Table(
 
 
 class ArcadeData(BaseData):
-    def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]:
+    async def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]:
         if serial is not None:
             serial = serial.replace("-", "")
             if len(serial) == 11:
@@ -89,12 +89,12 @@ class ArcadeData(BaseData):
             self.logger.error(f"{__name__ }: Need either serial or ID to look up!")
             return None
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_machine(
+    async def put_machine(
         self,
         arcade_id: int,
         serial: str = "",
@@ -110,13 +110,13 @@ class ArcadeData(BaseData):
             arcade=arcade_id, keychip=serial, board=board, game=game, is_cab=is_cab
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def set_machine_serial(self, machine_id: int, serial: str) -> None:
+    async def set_machine_serial(self, machine_id: int, serial: str) -> None:
-        result = self.execute(
+        result = await self.execute(
             machine.update(machine.c.id == machine_id).values(keychip=serial)
         )
         if result is None:
@@ -125,8 +125,8 @@ class ArcadeData(BaseData):
             )
         return result.lastrowid
 
-    def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
+    async def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
-        result = self.execute(
+        result = await self.execute(
             machine.update(machine.c.id == machine_id).values(board=boardid)
         )
         if result is None:
@@ -134,21 +134,21 @@ class ArcadeData(BaseData):
                 f"Failed to update board id for machine {machine_id} -> {boardid}"
             )
 
-    def get_arcade(self, id: int) -> Optional[Row]:
+    async def get_arcade(self, id: int) -> Optional[Row]:
         sql = arcade.select(arcade.c.id == id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
     
-    def get_arcade_machines(self, id: int) -> Optional[List[Row]]:
+    async def get_arcade_machines(self, id: int) -> Optional[List[Row]]:
         sql = machine.select(machine.c.arcade == id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_arcade(
+    async def put_arcade(
         self,
         name: str,
         nickname: str = None,
@@ -171,42 +171,42 @@ class ArcadeData(BaseData):
             regional_id=regional_id,
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_arcades_managed_by_user(self, user_id: int) -> Optional[List[Row]]:
+    async def get_arcades_managed_by_user(self, user_id: int) -> Optional[List[Row]]:
         sql = select(arcade).join(arcade_owner, arcade_owner.c.arcade == arcade.c.id).where(arcade_owner.c.user == user_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchall()
     
-    def get_manager_permissions(self, user_id: int, arcade_id: int) -> Optional[int]:
+    async def get_manager_permissions(self, user_id: int, arcade_id: int) -> Optional[int]:
         sql = select(arcade_owner.c.permissions).where(and_(arcade_owner.c.user == user_id, arcade_owner.c.arcade == arcade_id))
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchone()
 
-    def get_arcade_owners(self, arcade_id: int) -> Optional[Row]:
+    async def get_arcade_owners(self, arcade_id: int) -> Optional[Row]:
         sql = select(arcade_owner).where(arcade_owner.c.arcade == arcade_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
+    async def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
         sql = insert(arcade_owner).values(arcade=arcade_id, user=user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def format_serial(
+    async def format_serial(
         self, platform_code: str, platform_rev: int, serial_num: int, append: int = 4152
     ) -> str:
         return f"{platform_code}{platform_rev:02d}A{serial_num:04d}{append:04d}"  # 0x41 = A, 0x52 = R
@@ -217,16 +217,16 @@ class ArcadeData(BaseData):
         
         return True
 
-    def get_arcade_by_name(self, name: str) -> Optional[List[Row]]:
+    async def get_arcade_by_name(self, name: str) -> Optional[List[Row]]:
         sql = arcade.select(or_(arcade.c.name.like(f"%{name}%"), arcade.c.nickname.like(f"%{name}%")))
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_arcades_by_ip(self, ip: str) -> Optional[List[Row]]:
+    async def get_arcades_by_ip(self, ip: str) -> Optional[List[Row]]:
         sql = arcade.select().where(arcade.c.ip == ip)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()

core/data/schema/base.py

@@ -15,14 +15,6 @@ from core.config import CoreConfig
 
 metadata = MetaData()
 
-schema_ver = Table(
-    "schema_versions",
-    metadata,
-    Column("game", String(4), primary_key=True, nullable=False),
-    Column("version", Integer, nullable=False, server_default="1"),
-    mysql_charset="utf8mb4",
-)
-
 event_log = Table(
     "event_log",
     metadata,
@@ -43,11 +35,11 @@ class BaseData:
         self.conn = conn
         self.logger = logging.getLogger("database")
 
-    def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
+    async def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
         res = None
 
         try:
-            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}")
+            self.logger.debug(f"SQL Execute: {''.join(str(sql).splitlines())}")
             res = self.conn.execute(text(sql), opts)
 
         except SQLAlchemyError as e:
@@ -82,52 +74,7 @@ class BaseData:
         """
         return randrange(10000, 9999999)
 
-    def get_all_schema_vers(self) -> Optional[List[Row]]:
+    async def log_event(
-        sql = select(schema_ver)
-
-        result = self.execute(sql)
-        if result is None:
-            return None
-        return result.fetchall()
-
-    def get_schema_ver(self, game: str) -> Optional[int]:
-        sql = select(schema_ver).where(schema_ver.c.game == game)
-
-        result = self.execute(sql)
-        if result is None:
-            return None
-
-        row = result.fetchone()
-        if row is None:
-            return None
-
-        return row["version"]
-    
-    def touch_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version=schema_ver.c.version)
-
-        result = self.execute(conflict)
-        if result is None:
-            self.logger.error(
-                f"Failed to update schema version for game {game} (v{ver})"
-            )
-            return None
-        return result.lastrowid
-
-    def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version=ver)
-
-        result = self.execute(conflict)
-        if result is None:
-            self.logger.error(
-                f"Failed to update schema version for game {game} (v{ver})"
-            )
-            return None
-        return result.lastrowid
-
-    def log_event(
         self, system: str, type: str, severity: int, message: str, details: Dict = {}
     ) -> Optional[int]:
         sql = event_log.insert().values(
@@ -137,7 +84,7 @@ class BaseData:
             message=message,
             details=json.dumps(details),
         )
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             self.logger.error(
@@ -147,9 +94,9 @@ class BaseData:
 
         return result.lastrowid
 
-    def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
+    async def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
         sql = event_log.select().limit(entries).all()
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             return None

core/data/schema/card.py

@@ -27,87 +27,87 @@ aime_card = Table(
 
 
 class CardData(BaseData):
-    def get_card_by_access_code(self, access_code: str) -> Optional[Row]:
+    async def get_card_by_access_code(self, access_code: str) -> Optional[Row]:
         sql = aime_card.select(aime_card.c.access_code == access_code)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def get_card_by_id(self, card_id: int) -> Optional[Row]:
+    async def get_card_by_id(self, card_id: int) -> Optional[Row]:
         sql = aime_card.select(aime_card.c.id == card_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def update_access_code(self, old_ac: str, new_ac: str) -> None:
+    async def update_access_code(self, old_ac: str, new_ac: str) -> None:
         sql = aime_card.update(aime_card.c.access_code == old_ac).values(
             access_code=new_ac
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(
                 f"Failed to change card access code from {old_ac} to {new_ac}"
             )
 
-    def get_user_id_from_card(self, access_code: str) -> Optional[int]:
+    async def get_user_id_from_card(self, access_code: str) -> Optional[int]:
         """
         Given a 20 digit access code as a string, get the user id associated with that card
         """
-        card = self.get_card_by_access_code(access_code)
+        card = await self.get_card_by_access_code(access_code)
         if card is None:
             return None
 
         return int(card["user"])
 
-    def get_card_banned(self, access_code: str) -> Optional[bool]:
+    async def get_card_banned(self, access_code: str) -> Optional[bool]:
         """
         Given a 20 digit access code as a string, check if the card is banned
         """
-        card = self.get_card_by_access_code(access_code)
+        card = await self.get_card_by_access_code(access_code)
         if card is None:
             return None
         if card["is_banned"]:
             return True
         return False
-    def get_card_locked(self, access_code: str) -> Optional[bool]:
+    async def get_card_locked(self, access_code: str) -> Optional[bool]:
         """
         Given a 20 digit access code as a string, check if the card is locked
         """
-        card = self.get_card_by_access_code(access_code)
+        card = await self.get_card_by_access_code(access_code)
         if card is None:
             return None
         if card["is_locked"]:
             return True
         return False
 
-    def delete_card(self, card_id: int) -> None:
+    async def delete_card(self, card_id: int) -> None:
         sql = aime_card.delete(aime_card.c.id == card_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(f"Failed to delete card with id {card_id}")
 
-    def get_user_cards(self, aime_id: int) -> Optional[List[Row]]:
+    async def get_user_cards(self, aime_id: int) -> Optional[List[Row]]:
         """
         Returns all cards owned by a user
         """
         sql = aime_card.select(aime_card.c.user == aime_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def create_card(self, user_id: int, access_code: str) -> Optional[int]:
+    async def create_card(self, user_id: int, access_code: str) -> Optional[int]:
         """
         Given a aime_user id and a 20 digit access code as a string, create a card and return the ID if successful
         """
         sql = aime_card.insert().values(user=user_id, access_code=access_code)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid

core/data/schema/user.py

@@ -1,4 +1,3 @@
-from enum import Enum
 from typing import Optional, List
 from sqlalchemy import Table, Column
 from sqlalchemy.types import Integer, String, TIMESTAMP
@@ -24,15 +23,8 @@ aime_user = Table(
     mysql_charset="utf8mb4",
 )
 
-
-class PermissionBits(Enum):
-    PermUser = 1
-    PermMod = 2
-    PermSysAdmin = 4
-
-
 class UserData(BaseData):
-    def create_user(
+    async def create_user(
         self,
         id: int = None,
         username: str = None,
@@ -60,20 +52,20 @@ class UserData(BaseData):
             username=username, email=email, password=password, permissions=permission
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_user(self, user_id: int) -> Optional[Row]:
+    async def get_user(self, user_id: int) -> Optional[Row]:
         sql = select(aime_user).where(aime_user.c.id == user_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchone()
 
-    def check_password(self, user_id: int, passwd: bytes = None) -> bool:
+    async def check_password(self, user_id: int, passwd: bytes = None) -> bool:
-        usr = self.get_user(user_id)
+        usr = await self.get_user(user_id)
         if usr is None:
             return False
 
@@ -85,39 +77,34 @@ class UserData(BaseData):
 
         return bcrypt.checkpw(passwd, usr["password"].encode())
 
-    def reset_autoincrement(self, ai_value: int) -> None:
+    async def delete_user(self, user_id: int) -> None:
-        # ALTER TABLE isn't in sqlalchemy so we do this the ugly way
-        sql = f"ALTER TABLE aime_user AUTO_INCREMENT={ai_value}"
-        self.execute(sql)
-
-    def delete_user(self, user_id: int) -> None:
         sql = aime_user.delete(aime_user.c.id == user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(f"Failed to delete user with id {user_id}")
 
-    def get_unregistered_users(self) -> List[Row]:
+    async def get_unregistered_users(self) -> List[Row]:
         """
         Returns a list of users who have not registered with the webui. They may or may not have cards.
         """
         sql = select(aime_user).where(aime_user.c.password == None)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def find_user_by_email(self, email: str) -> Row:
+    async def find_user_by_email(self, email: str) -> Row:
         sql = select(aime_user).where(aime_user.c.email == email)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchone()
 
-    def find_user_by_username(self, username: str) -> List[Row]:
+    async def find_user_by_username(self, username: str) -> List[Row]:
         sql = aime_user.select(aime_user.c.username.like(f"%{username}%"))
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchall()

core/frontend/arcade/index.jinja

@@ -1,4 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-<h1>{{ arcade.name }}</h1>
-{% endblock content %}

core/frontend/machine/index.jinja

@@ -1,5 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-{% include "core/frontend/widgets/err_banner.jinja" %}
-<h1>Machine Management</h1>
-{% endblock content %}

core/frontend/sys/index.jinja

@@ -1,103 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-<h1>System Management</h1>
-
-<div class="row" id="rowForm">
-    {% if sesh.permissions >= 2 %}
-    <div class="col-sm-6" style="max-width: 25%;">
-        <form id="usrLookup" name="usrLookup" action="/sys/lookup.user" class="form-inline">
-            <h3>User Search</h3>
-            <div class="form-group">
-                <label for="usrId">User ID</label>
-                <input type="number" class="form-control" id="usrId" name="usrId">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="usrName">Username</label>
-                <input type="text" class="form-control" id="usrName" name="usrName">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="usrEmail">Email address</label>
-                <input type="email" class="form-control" id="usrEmail" name="usrEmail" aria-describedby="emailHelp">
-            </div>
-            <br />
-            <button type="submit" class="btn btn-primary">Search</button>
-        </form>
-    </div>
-    {% endif %}
-    {% if sesh.permissions >= 4 %}
-    <div class="col-sm-6" style="max-width: 25%;">
-        <form id="arcadeLookup" name="arcadeLookup" action="/sys/lookup.arcade" class="form-inline" >
-            <h3>Arcade Search</h3>
-            <div class="form-group">
-                <label for="arcadeId">Arcade ID</label>
-                <input type="number" class="form-control" id="arcadeId" name="arcadeId">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="arcadeName">Arcade Name</label>
-                <input type="text" class="form-control" id="arcadeName" name="arcadeName">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="arcadeUser">Owner User ID</label>
-                <input type="number" class="form-control" id="arcadeUser" name="arcadeUser">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="arcadeIp">Assigned IP Address</label>
-                <input type="text" class="form-control" id="arcadeIp" name="arcadeIp">
-            </div>
-            <br />
-            <button type="submit" class="btn btn-primary">Search</button>
-        </form>
-    </div>
-    <div class="col-sm-6" style="max-width: 25%;">
-        <form id="cabLookup" name="cabLookup" action="/sys/lookup.cab" class="form-inline" >
-            <h3>Machine Search</h3>
-            <div class="form-group">
-                <label for="cabId">Machine ID</label>
-                <input type="number" class="form-control" id="cabId" name="cabId">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="cabSerial">Machine Serial</label>
-                <input type="text" class="form-control" id="cabSerial" name="cabSerial">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="cabAcId">Arcade ID</label>
-                <input type="number" class="form-control" id="cabAcId" name="cabAcId">
-            </div>
-            <br />
-            <button type="submit" class="btn btn-primary">Search</button>
-        </form>
-    </div>
-    {% endif %}
-</div>
-<div class="row" id="rowResult" style="margin: 10px;">
-    {% if sesh.permissions >= 2 %}
-    <div id="userSearchResult" class="col-sm-6" style="max-width: 25%;">
-        {% for usr in usrlist %}
-        <a href=/user/{{ usr.id }}><pre>{{ usr.id }} | {{ usr.username if usr.username != None else "<i>No Name Set</i>"}}</pre></a>
-        {% endfor %}
-    </div>
-    {% endif %}
-    {% if sesh.permissions >= 4 %}
-    <div id="arcadeSearchResult" class="col-sm-6" style="max-width: 25%;">
-        {% for ac in aclist %}
-        <pre><a href=/arcade/{{ ac.id }}>{{ ac.id }} | {{ ac.name if ac.name != None else "<i>No Name Set</i>" }} | {{ ac.ip if ac.ip != None else "<i>No IP Assigned</i>"}}</pre></a>
-        {% endfor %}
-    </div
-    ><div id="cabSearchResult" class="col-sm-6" style="max-width: 25%;">
-        {% for cab in cablist %}
-        <a href=/cab/{{ cab.id }}><pre>{{ cab.id }} | {{ cab.game if cab.game != None else "<i>ANY </i>" }} | {{ cab.serial }}</pre></a>
-        {% endfor %}
-    </div>
-    {% endif %}
-</div>
-<div class="row" id="rowAdd">
-
-</div>
-{% endblock content %}

core/frontend/user/index.jinja

@@ -1,41 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-<h1>Management for {{ username }}</h1>
-<h2>Cards <button class="btn btn-success" data-bs-toggle="modal" data-bs-target="#card_add">Add</button></h2>
-<ul style="font-size: 20px;">
-{% for c in cards %}
-<li>{{ c.access_code }}: {{ c.status }}&nbsp;{% if c.status == 'Active'%}<button class="btn-warning btn">Lock</button>{% elif c.status == 'Locked' %}<button class="btn-warning btn">Unlock</button>{% endif %}&nbsp;<button class="btn-danger btn">Delete</button></li>
-{% endfor %}
-</ul>
-
-{% if arcades is defined %}
-<h2>Arcades</h2>
-  <ul style="font-size: 20px;">
-  {% for a in arcades %}
-    <li><a href=/arcade/{{ a.id }}>{{ a.name }}</a></li>
-  {% endfor %}
-  </ul>
-{% endif %}
-
-<div class="modal fade" id="card_add" tabindex="-1" aria-labelledby="card_add_label" aria-hidden="true">
-  <div class="modal-dialog">
-    <div class="modal-content">
-      <div class="modal-header">
-        <h1 class="modal-title fs-5" id="card_add_label">Add Card</h1>
-        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
-      </div>
-      <div class="modal-body">
-        HOW TO:<br>
-        Scan your card on any networked game and press the "View Access Code" button (varies by game) and enter the 20 digit code below.<br>
-        !!FOR AMUSEIC CARDS: DO NOT ENTER THE CODE SHOWN ON THE BACK OF THE CARD ITSELF OR IT WILL NOT WORK!!
-        <p /><label for="card_add_frm_access_code">Access Code:&nbsp;</label><input id="card_add_frm_access_code" maxlength="20" type="text" required> 
-      </div>
-      <div class="modal-footer">
-        <button type="button" class="btn btn-primary">Add</button>
-        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
-      </div>
-    </div>
-  </div>
-</div>
-
-{% endblock content %}

core/frontend/widgets/err_banner.jinja

@@ -1,18 +0,0 @@
-{% if error > 0 %}
-<div class="err-banner">
-<h3>Error</h3>
-{% if error == 1 %}
-Card not registered, or wrong password
-{% elif error == 2 %}
-Missing or malformed access code
-{% elif error == 3 %}
-Failed to create user
-{% elif error == 4 %}
-Arcade not found
-{% elif error == 5 %}
-Machine not found
-{% else %}
-An unknown error occoured
-{% endif %}
-</div>
-{% endif %}

core/mucha.py

@@ -1,8 +1,7 @@
-from typing import Dict, Any, Optional, List
+from typing import Dict, Any, Optional
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
-from twisted.web import resource
+from starlette.requests import Request
-from twisted.web.http import Request
 from datetime import datetime
 from Crypto.Cipher import Blowfish
 import pytz
@@ -12,7 +11,7 @@ from .utils import Utils
 from .title import TitleServlet
 
 class MuchaServlet:
-    mucha_registry: List[str] = []
+    mucha_registry: Dict[str, str] = {}
     def __init__(self, cfg: CoreConfig, cfg_dir: str) -> None:
         self.config = cfg
         self.config_dir = cfg_dir
@@ -39,11 +38,12 @@ class MuchaServlet:
 
         for _, mod in TitleServlet.title_registry.items():
             if hasattr(mod, "get_mucha_info"):
-                enabled, game_cd = mod.get_mucha_info(
+                enabled, game_cds, netid_prefixes = mod.get_mucha_info(
                     self.config, self.config_dir
                 )
                 if enabled:
-                    self.mucha_registry.append(game_cd)
+                    for x in range(len(game_cds)):
+                        self.mucha_registry[game_cds[x]] = netid_prefixes[x]
 
         self.logger.info(f"Serving {len(self.mucha_registry)} games")
 
@@ -75,7 +75,7 @@ class MuchaServlet:
         self.logger.debug(f"Decrypt SN to {sn_decrypt.hex()}")
 
         resp = MuchaAuthResponse(
-            f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}"
+            f"{self.config.server.hostname}{':' + str(self.config.server.port) if self.config.server.is_develop else ''}"
         )
 
         self.logger.debug(f"Mucha response {vars(resp)}")
@@ -100,7 +100,7 @@ class MuchaServlet:
             self.logger.warning(f"Unknown gameCd {req.gameCd}")
             return b"RESULTS=000"
 
-        resp = MuchaUpdateResponse(req.gameVer, f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}")
+        resp = MuchaUpdateResponse(req.gameVer, f"{self.config.server.hostname}{':' + str(self.config.server.port) if self.config.server.is_develop else ''}")
 
         self.logger.debug(f"Mucha response {vars(resp)}")
 

core/templates/arcade/index.jinja

@@ -0,0 +1,19 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+{% if arcade is defined %}
+<h1>{{ arcade.name }}</h1>
+<h2>PCBs assigned to this arcade <button class="btn btn-success" id="btn_add_cab" onclick="toggle_add_cab_form()">Add</button></h2>
+{% if success is defined and success == 3 %}
+<div style="background-color: #00AA00; padding: 20px; margin-bottom: 10px; width: 15%;">
+Cab added successfully
+</div>
+{% endif %}
+<ul style="font-size: 20px;">
+{% for c in arcade.cabs %}
+<li><a href="/cab/{{ c.id }}">{{ c.serial }} ({{ c.game }})</a>&nbsp;<button class="btn btn-secondary" onclick="prep_edit_form()">Edit</button>&nbsp;<button class="btn-danger btn">Delete</button></li>
+{% endfor %}
+</ul>
+{% else %}
+<h3>Arcade Not Found</h3>
+{% endif %}
+{% endblock content %}

core/templates/gate/create.jinja

@@ -1,4 +1,4 @@
-{% extends "core/frontend/index.jinja" %}
+{% extends "core/templates/index.jinja" %}
 {% block content %}
 <h1>Create User</h1>
 <form id="create" style="max-width: 240px; min-width: 10%;" action="/gate/gate.create" method="post">

core/templates/gate/gate.jinja

@@ -1,7 +1,7 @@
-{% extends "core/frontend/index.jinja" %}
+{% extends "core/templates/index.jinja" %}
 {% block content %}
 <h1>Gate</h1>
-{% include "core/frontend/widgets/err_banner.jinja" %}
+{% include "core/templates/widgets/err_banner.jinja" %}
 <style>
     /* Chrome, Safari, Edge, Opera */
     input::-webkit-outer-spin-button,

core/templates/index.jinja

@@ -84,7 +84,7 @@
         </style>
     </head>
     <body>
-    {% include "core/frontend/widgets/topbar.jinja" %}
+    {% include "core/templates/widgets/topbar.jinja" %}
     {% block content %}
     <h1>{{ server_name }}</h1>
     {% endblock content %}

core/templates/machine/index.jinja

@@ -0,0 +1,4 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+<h1>Machine Management</h1>
+{% endblock content %}

core/templates/sys/index.jinja

@@ -0,0 +1,69 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+<h1>System Management</h1>
+{% if error is defined %}
+{% include "core/templates/widgets/err_banner.jinja" %}
+{% endif %}
+<div class="row" id="rowForm">
+    {% if "{:08b}".format(sesh.permissions)[6] == "1" %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="usrLookup" name="usrLookup" action="/sys/lookup.user" class="form-inline">
+            <h3>User Search</h3>
+            <div class="form-group">
+                <label for="usrId">User ID</label>
+                <input type="number" class="form-control" id="usrId" name="usrId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrName">Username</label>
+                <input type="text" class="form-control" id="usrName" name="usrName">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrEmail">Email address</label>
+                <input type="email" class="form-control" id="usrEmail" name="usrEmail" aria-describedby="emailHelp">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    {% endif %}
+    {% if "{:08b}".format(sesh.permissions)[5] == "1" %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="shopLookup" name="shopLookup" action="/sys/lookup.shop" class="form-inline">
+            <h3>Shop search</h3>
+            <div class="form-group">
+                <label for="shopId">Shop ID</label>
+                <input type="number" class="form-control" id="shopId" name="shopId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="serialNum">Serial Number</label>
+                <input type="text" class="form-control" id="serialNum" name="serialNum" maxlength="15">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    {% endif %}
+</div>
+<div class="row" id="rowResult" style="margin: 10px;">
+    {% if "{:08b}".format(sesh.permissions)[6] == "1" %}
+    <div id="userSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for usr in usrlist %}
+        <a href=/user/{{ usr.id }}><pre>{{ usr.username if usr.username is not none else "<i>No Name Set</i>"}}</pre></a>
+        {% endfor %}
+    </div>
+    {% endif %}
+    {% if "{:08b}".format(sesh.permissions)[5] == "1" %}
+    <div id="shopSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for shop in shoplist %}
+        <a href="/shop/{{ shop.id }}"><pre>{{ shop.name if shop.name else "<i>No Name Set</i>"}}</pre></a>
+        {% endfor %}
+    </div>
+    {% endif %}
+</div>
+<div class="row" id="rowAdd">
+
+</div>
+{% endblock content %}

core/templates/widgets/err_banner.jinja

@@ -0,0 +1,29 @@
+{% if error > 0 %}
+<div class="err-banner">
+<h3>Error</h3>
+{% if error == 1 %}
+Card not registered, or wrong password
+{% elif error == 2 %}
+Missing or malformed access code
+{% elif error == 3 %}
+Failed to create user
+{% elif error == 4 %}
+Required field not filled or invalid
+{% elif error == 5 %}
+Incorrect old password
+{% elif error == 6 %}
+Passwords don't match
+{% elif error == 7 %}
+New password not acceptable
+{% elif error == 8 %}
+New Nickname too long
+{% elif error == 9 %}
+You must be logged in to preform this action
+New Nickname too long
+{% elif error == 10 %}
+Invalid serial number
+{% else %}
+An unknown error occoured
+{% endif %}
+</div>
+{% endif %}

core/templates/widgets/topbar.jinja

@@ -3,19 +3,20 @@
 </div>
 <div style="background: #333; color: #f9f9f9; width: 80%; height: 50px; line-height: 50px; padding-left: 10px; float: left;">
     <a href=/><button class="btn btn-primary">Home</button></a>&nbsp;
-    {% for game in game_list %}    
+    {% for game, data in game_list|items %}    
-    <a href=/game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
+    <a href=/game{{ data.url }}/><button class="btn btn-success">{{ game }}</button></a>&nbsp;
     {% endfor %}
 </div>
 </div>
 <div style="background: #333; color: #f9f9f9; width: 10%; height: 50px; line-height: 50px; text-align: center; float: left;">
     {% if sesh is defined and sesh["permissions"] >= 2 %}
-    <a href="/sys"><button class="btn btn-primary">System</button></a>
+    <a href="/sys/"><button class="btn btn-primary">System</button></a>
     {% endif %}
-    {% if sesh is defined and sesh["userId"] > 0 %}
+    {% if sesh is defined and sesh["user_id"] > 0 %}
-    <a href="/user"><button class="btn btn-primary">Account</button></a>
+    <a href="/user/"><button class="btn btn-primary">Account</button></a>
+    <a href="/user/logout"><button class="btn btn-danger">Logout</button></a>
     {% else %}
-    <a href="/gate"><button class="btn btn-primary">Gate</button></a>
+    <a href="/gate/"><button class="btn btn-primary">Gate</button></a>
     {% endif %}
     
 </div>

core/title.py

@@ -1,12 +1,24 @@
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Any
+import json
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
-from twisted.web.http import Request
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Route
 
 from core.config import CoreConfig
 from core.data import Data
 from core.utils import Utils
 
+class JSONResponseNoASCII(Response):
+    media_type = "application/json"
+
+    def render(self, content: Any) -> bytes:
+        return json.dumps(
+            content,
+            ensure_ascii=False,
+        ).encode("utf-8")
+
 class BaseServlet:
     def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
         self.core_cfg = core_cfg
@@ -28,18 +40,16 @@ class BaseServlet:
         """
         return False
     
-    def get_endpoint_matchers(self) -> Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]:
+    def get_routes(self) -> List[Route]:
         """Called during boot to get all matcher endpoints this title servlet handles
 
         Returns:
-            Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]: A 2-length tuple where offset 0 is GET and offset 1 is POST, 
+            List[Route]: A list of Routes, WebSocketRoutes, or similar classes
-            containing a list of 3-length tuples where offset 0 is the name of the function in the handler that should be called, offset 1
-            is the matching string, and offset 2 is a dict containing rules for the matcher.
         """
-        return (
+        return [
-            [("render_GET", "/{game}/{version}/{endpoint}", {'game': R'S...'})], 
+            Route("/{game}/{version}/{endpoint}", self.render_POST, methods=["POST"]),
-            [("render_POST", "/{game}/{version}/{endpoint}", {'game': R'S...'})]
+            Route("/{game}/{version}/{endpoint}", self.render_GET, methods=["GET"]),
-        )
+        ]
     
     def setup(self) -> None:
         """Called once during boot, should contain any additional setup the handler must do, such as starting any sub-services
@@ -58,11 +68,11 @@ class BaseServlet:
             Tuple[str, str]: A tuple where offset 0 is the allnet uri field, and offset 1 is the allnet host field
         """
         if not self.core_cfg.server.is_using_proxy and Utils.get_title_port(self.core_cfg) != 80:
-            return (f"http://{self.core_cfg.title.hostname}:{Utils.get_title_port(self.core_cfg)}/{game_code}/{game_ver}/", "")
+            return (f"http://{self.core_cfg.server.hostname}:{Utils.get_title_port(self.core_cfg)}/{game_code}/{game_ver}/", "")
 
-        return (f"http://{self.core_cfg.title.hostname}/{game_code}/{game_ver}/", "")
+        return (f"http://{self.core_cfg.server.hostname}/{game_code}/{game_ver}/", "")
 
-    def get_mucha_info(self, core_cfg: CoreConfig, cfg_dir: str) -> Tuple[bool, str]:
+    def get_mucha_info(self, core_cfg: CoreConfig, cfg_dir: str) -> Tuple[bool, List[str], List[str]]:
         """Called once during boot to check if this game is a mucha game
 
         Args:
@@ -72,15 +82,15 @@ class BaseServlet:
         Returns:
             Tuple[bool, str]: Tuple where offset 0 is true if the game is enabled, false otherwise, and offset 1 is the game CD
         """
-        return (False, "")
+        return (False, [], [])
 
-    def render_POST(self, request: Request, game_code: str, matchers: Dict) -> bytes:
+    async def render_POST(self, request: Request) -> bytes:
-        self.logger.warn(f"{game_code} Does not dispatch POST")
+        self.logger.warn(f"Game Does not dispatch POST")
-        return None
+        return Response()
 
-    def render_GET(self, request: Request, game_code: str, matchers: Dict) -> bytes:
+    async def render_GET(self, request: Request) -> bytes:
-        self.logger.warn(f"{game_code} Does not dispatch GET")
+        self.logger.warn(f"Game Does not dispatch GET")
-        return None
+        return Response()
 
 class TitleServlet:
     title_registry: Dict[str, BaseServlet] = {}
@@ -136,7 +146,7 @@ class TitleServlet:
                 self.logger.error(f"{folder} missing game_code or index in __init__.py, or is_game_enabled in index")
 
         self.logger.info(
-            f"Serving {len(self.title_registry)} game codes {'on port ' + str(core_cfg.title.port) if core_cfg.title.port > 0 else ''}"
+            f"Serving {len(self.title_registry)} game codes {'on port ' + str(core_cfg.server.port) if core_cfg.server.port > 0 else ''}"
         )
 
     def render_GET(self, request: Request, endpoints: dict) -> bytes:

core/utils.py

@@ -1,6 +1,6 @@
 from typing import Dict, Any, Optional
 from types import ModuleType
-from twisted.web.http import Request
+from starlette.requests import Request
 import logging
 import importlib
 from os import walk
@@ -34,21 +34,13 @@ class Utils:
 
     @classmethod
     def get_ip_addr(cls, req: Request) -> str:
-        return (
+        return req.headers.get("x-forwarded-for", req.client.host)
-            req.getAllHeaders()[b"x-forwarded-for"].decode()
-            if b"x-forwarded-for" in req.getAllHeaders()
-            else req.getClientAddress().host
-        )
     
     @classmethod
     def get_title_port(cls, cfg: CoreConfig):
         if cls.real_title_port is not None: return cls.real_title_port
 
-        if cfg.title.port == 0:
+        cls.real_title_port = cfg.server.proxy_port if cfg.server.is_using_proxy and cfg.server.proxy_port else cfg.server.port
-            cls.real_title_port = cfg.allnet.port
-        
-        else:
-            cls.real_title_port = cfg.title.port
         
         return cls.real_title_port
     
@@ -56,11 +48,7 @@ class Utils:
     def get_title_port_ssl(cls, cfg: CoreConfig):
         if cls.real_title_port_ssl is not None: return cls.real_title_port_ssl
 
-        if cfg.title.port_ssl == 0:
+        cls.real_title_port_ssl = cfg.server.proxy_port_ssl if cfg.server.is_using_proxy and cfg.server.proxy_port_ssl else Utils.get_title_port(cfg)
-            cls.real_title_port_ssl = 443
-        
-        else:
-            cls.real_title_port_ssl = cfg.title.port_ssl
         
         return cls.real_title_port_ssl
 

dbutils.py

@@ -1,9 +1,12 @@
-import yaml
+#!/usr/bin/env python3
 import argparse
 import logging
-from core.config import CoreConfig
+from os import mkdir, path, access, W_OK
+import yaml
+import asyncio
+
 from core.data import Data
-from os import path, mkdir, access, W_OK
+from core.config import CoreConfig
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Database utilities")
@@ -16,19 +19,9 @@ if __name__ == "__main__":
         type=str,
         help="Version of the database to upgrade/rollback to",
     )
-    parser.add_argument(
-        "--game",
-        "-g",
-        type=str,
-        help="Game code of the game who's schema will be updated/rolled back. Ex. SDFE",
-    )
     parser.add_argument("--email", "-e", type=str, help="Email for the new user")
-    parser.add_argument("--old_ac", "-o", type=str, help="Access code to transfer from")
+    parser.add_argument("--access_code", "-a", type=str, help="Access code for new/transfer user", default="00000000000000000000")
-    parser.add_argument("--new_ac", "-n", type=str, help="Access code to transfer to")
+    parser.add_argument("action", type=str, help="create, upgrade, create-owner")
-    parser.add_argument("--force", "-f", type=bool, help="Force the action to happen")
-    parser.add_argument(
-        "action", type=str, help="DB Action, create, recreate, upgrade, or rollback"
-    )
     args = parser.parse_args()
 
     cfg = CoreConfig()
@@ -51,41 +44,17 @@ if __name__ == "__main__":
     if args.action == "create":
         data.create_database()
     
-    elif args.action == "recreate":
+    elif args.action == "upgrade":
-        data.recreate_database()
+        data.schema_upgrade(args.version)
-
-    elif args.action == "upgrade" or args.action == "rollback":
-        if args.version is None:
-            data.logger.warning("No version set, upgrading to latest")
-
-        if args.game is None:
-            data.logger.warning("No game set, upgrading core schema")
-            data.migrate_database(
-                "CORE",
-                int(args.version) if args.version is not None else None,
-                args.action,
-            )
-
-        else:
-            data.migrate_database(
-                args.game,
-                int(args.version) if args.version is not None else None,
-                args.action,
-            )
-
-    elif args.action == "autoupgrade":
-        data.autoupgrade()
 
     elif args.action == "create-owner":
-        data.create_owner(args.email)
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.create_owner(args.email, args.access_code))
+        data.schema_upgrade(args.version)
 
-    elif args.action == "migrate-card":
+    elif args.action == "migrate":
-        data.migrate_card(args.old_ac, args.new_ac, args.force)
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.migrate())
 
-    elif args.action == "cleanup":
+    else:
-        data.delete_hanging_users()
+        logging.getLogger("database").info(f"Unknown action {args.action}")
-    
-    elif args.action == "version":
-        data.show_versions()
-
-    data.logger.info("Done")

docs/config.md

@@ -1,23 +1,24 @@
 # ARTEMiS Configuration
 ## Server
 - `listen_address`: IP Address or hostname that the server will listen for connections on. Set to 127.0.0.1 for local only, or 0.0.0.0 for all interfaces. Default `127.0.0.1`
+- `hostname`: Hostname that gets sent to clients to tell them where to connect. Games must be able to connect to your server via the hostname or IP you spcify here. Note that most games will reject `localhost` or `127.0.0.1`. Default `localhost`
+- `port`: Port that the server will listen for connections on. Default `80`
+- `ssl_key`: Location of the ssl server key for the secure title server. Ignored if you don't use SSL. Default `cert/title.key`
+- `ssl_cert`: Location of the ssl server certificate for the secure title server. Must not be a self-signed SSL. Ignored if you don't use SSL. Default `cert/title.pem`
 - `allow_user_registration`: Allows users to register in-game via the AimeDB `register` function. Disable to be able to control who can use cards on your server. Default `True`
 - `allow_unregistered_serials`: Allows games that do not have registered keychips to connect and authenticate. Disable to restrict who can connect to your server. Recomended to disable for production setups. Default `True`
 - `name`: Name for the server, used by some games in their default MOTDs. Default `ARTEMiS`
 - `is_develop`: Flags that the server is a development instance without a proxy standing in front of it. Setting to `False` tells the server not to listen for SSL, because the proxy should be handling all SSL-related things, among other things. Default `True`
-- `threading`: Flags that `reactor.run` should be called via the `Thread` standard library. May provide a speed boost, but removes the ability to kill the server via `Ctrl + C`. Default: `False`
+- `is_using_proxy`: Flags that you'll be using some other software, such as nginx, to proxy requests, and to send `proxy_port` or `proxy_port_ssl` to games instead of `port`. Default `False`
-- `check_arcade_ip`: Checks IPs against the `arcade` table in the database, if one is defined. Default `False`
+- `proxy_port`: Which port your front-facing proxy will be listening on. Ignored if `is_using_proxy` is `False` or if set to `0`. Default `0`
-- `strict_ip_checking`: Rejects clients if there is no IP in the `arcade` table for the respective arcade
+- `proxy_port`: Which port your front-facing proxy will be listening for ssl connections on. Ignored if `is_using_proxy` is `False` or if set to `0`. Default `0`
 - `log_dir`: Directory to store logs. Server MUST have read and write permissions to this directory or you will have issues. Default `logs`
+- `check_arcade_ip`: Checks IPs against the `arcade` table in the database, if one is defined. Default `False`
+- `strict_ip_checking`: Rejects clients if there is no IP in the `arcade` table for the respective arcade. Default `False`
 ## Title
 - `loglevel`: Logging level for the title server. Default `info`
-- `hostname`: Hostname that gets sent to clients to tell them where to connect. Games must be able to connect to your server via the hostname or IP you spcify here. Note that most games will reject `localhost` or `127.0.0.1`. Default `localhost`
+- `reboot_start_time`: 24 hour JST time that clients will see as the start of maintenance period, ex `04:00`. Leave blank for no maintenance time. Default: `""`
-- `port`: Port that the title server will listen for connections on. Set to 0 to use the Allnet handler to reduce the port footprint. Default `8080`
+- `reboot_end_time`: 24 hour JST time that clients will see as the end of maintenance period, ex `05:00`. Leave blank for no maintenance time. Default: `""`
-- `port_ssl`: Port that the secure title server will listen for connections on. Set to 0 to use the Allnet handler to reduce the port footprint. Default `0`
-- `ssl_key`: Location of the ssl server key for the secure title server. Ignored if `port_ssl` is set to `0` or `is_develop` set to `False`. Default `cert/title.key`
-- `ssl_cert`: Location of the ssl server certificate for the secure title server. Must not be a self-signed SSL. Ignored if `port_ssl` is set to `0` or `is_develop` is set to `False`. Default `cert/title.pem`
-- `reboot_start_time`: 24 hour JST time that clients will see as the start of maintenance period. Leave blank for no maintenance time. Default: ""
-- `reboot_end_time`: 24 hour JST time that clients will see as the end of maintenance period. Leave blank for no maintenance time. Default: ""
 ## Database
 - `host`: Host of the database. Default `localhost`
 - `username`: Username of the account the server should connect to the database with. Default `aime`
@@ -26,23 +27,30 @@
 - `port`: Port the database server is listening on. Default `3306`
 - `protocol`: Protocol used in the connection string, e.i `mysql` would result in `mysql://...`. Default `mysql`
 - `sha2_password`: Weather or not the password in the connection string should be hashed via SHA2. Default `False`
-- `loglevel`: Logging level for the database. Default `warn`
+- `loglevel`: Logging level for the database. Default `info`
-- `user_table_autoincrement_start`: What the `aime_user` table ID autoincrememnt should start with. Default `10000`
 - `memcached_host`: Host of the memcached server. Default `localhost`
 ## Frontend
-- `enable`: Weather or not the frontend should be enabled. Default `False`
+- `enable`: Weather or not the frontend should run. Default `False`
-- `port`: Port the frontend should listen for connections on. Default `8090`
+- `port`: Port the frontend should listen on. Default `8080`
 - `loglevel`: Logging level for the frontend server. Default `info`
+- `secret`: Base64-encoded JWT secret for session cookies, generated by you. Default `""`
 ## Allnet
+- `standalone`: Weather allnet should be launched as a standalone service on it's own port.
+- `port`: Port the billing server should listen for connections on. Games are hardcoded to ask for port `80` so only change if you have a proxy redirecting properly. Default `80`
 - `loglevel`: Logging level for the allnet server. Default `info`
-- `port`: Port the allnet server should listen for connections on. Games are hardcoded to ask for port `80` so only change if you have a proxy redirecting properly. Default `80`
 - `allow_online_updates`: Allow allnet to distribute online updates via DownloadOrders. This system is currently non-functional, so leave it disabled. Default `False`
+- `update_cfg_folder`: Folder where delivery INI files will be checked for. Ignored if `allow_online_updates` is `False`. Default `""`
 ## Billing
-- `port`: Port the billing server should listen for connections on. Games are hardcoded to ask for port `8443` so only change if you have a proxy redirecting properly. Set to 0 to use the allnet handler to reduce the number of ports the server eats up. Default `8443`
+- `standalone`: Weather or not the billing server should be launched as a standalone service on it's own port. Setting this to `True` requires that you have `ssl_key` and `ssl_cert` set. Default `False`
-- `ssl_key`: Location of the ssl server key for the billing server. Ignored if `port` is set to `0` or `is_develop` set to `False`. Default `cert/server.key`
+- `loglevel`: Logging level for the billing server. Default `info`
-- `ssl_cert`: Location of the ssl server certificate for the billing server. Must match the CA distributed to users or the billing server will not connect. Ignored if `port` is set to `0` or `is_develop` is set to `False`. Default `cert/server.pem`
+- `port`: Port the billing server should listen for connections on. Games are hardcoded to ask for port `8443` so only change if you have a proxy redirecting properly. Ignored if `standalone` is `False`. Default `8443`
+- `ssl_key`: Location of the ssl server key for the billing server. Ignored if `standalone` is `False`. Default `cert/server.key`
+- `ssl_cert`: Location of the ssl server certificate for the billing server. Ignored if `standalone` is `False`.  Must match the CA distributed to users or the billing server will not connect. Default `cert/server.pem`
 - `signing_key`: Location of the RSA Private key used to sign billing requests. Must match the public key distributed to users or the billing server will not connect. Default `cert/billing.key`
 ## Aimedb
+- `enable`: Weather or not aimedb should run. Default `True`
 - `loglevel`: Logging level for the aimedb server. Default `info`
 - `port`: Port the aimedb server should listen for connections on. Games are hardcoded to ask for port `22345` so only change if you have a proxy redirecting properly. Default `22345`
 - `key`: Key to encrypt/decrypt aimedb requests and responses. MUST be set or the server will not start. If set incorrectly, your server will not properly handle aimedb requests. Default `""`
+- `id_secret`: Base64-encoded JWT secret for Sega Auth IDs. Leaving this blank disables this feature. Default `""`
+- `id_lifetime_seconds`: Number of secons a JWT generated should be valid for. Default `86400` (1 day)

example_config/core.yaml

@@ -1,26 +1,25 @@
 server:
   listen_address: "127.0.0.1"  
+  hostname: "localhost"  
+  port: 80
+  ssl_key: "cert/title.key"
+  ssl_cert: "cert/title.crt"
   allow_user_registration: True
   allow_unregistered_serials: True
   name: "ARTEMiS"
   is_develop: True
   is_using_proxy: False
-  threading: False
+  proxy_port: 0
+  proxy_port_ssl: 0
   log_dir: "logs"
   check_arcade_ip: False
   strict_ip_checking: False
 
 title:
   loglevel: "info"
-  hostname: "localhost"
-  port: 8080
-  port_ssl: 0
-  ssl_cert: "cert/title.crt"
-  ssl_key: "cert/title.key"
   reboot_start_time: "04:00"
   reboot_end_time: "05:00"
 
-
 database:
   host: "localhost"
   username: "aime"
@@ -29,30 +28,33 @@ database:
   port: 3306
   protocol: "mysql"
   sha2_password: False
-  loglevel: "warn"
+  loglevel: "info"
-  user_table_autoincrement_start: 10000
   enable_memcached: True
   memcached_host: "localhost"
 
 frontend:
-  enable: False
+  enable: True
-  port: 8090
+  port: 8080
   loglevel: "info"
+  secret: ""
 
 allnet:
-  loglevel: "info"
+  standalone: False
   port: 80
-  ip_check: False
+  loglevel: "info"
   allow_online_updates: False
   update_cfg_folder: ""
 
 billing:
+  standalone: True
+  loglevel: "info"
   port: 8443
   ssl_key: "cert/server.key"
   ssl_cert: "cert/server.pem"
   signing_key: "cert/billing.key"
 
 aimedb:
+  enable: True
   loglevel: "info"
   port: 22345
   key: ""
@@ -60,6 +62,4 @@ aimedb:
   id_lifetime_seconds: 86400
 
 mucha:
-  enable: False
-  hostname: "localhost"
   loglevel: "info"

example_config/cxb.yaml

@@ -1,3 +1,4 @@
 server:
   enable: True
   loglevel: "info"
+  use:https: True

example_config/nginx_example.conf

@@ -6,7 +6,7 @@ server {
 	location / {
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 		proxy_pass_request_headers on;
-		proxy_pass http://localhost:8000/;
+		proxy_pass http://localhost:8080/;
 	}
 }
 
@@ -42,7 +42,7 @@ server {
 	}
 }
 
-# Billing
+# Billing, comment this out if running billing standalone
 server {
 	listen 8443 ssl;	
 	server_name ib.naominet.jp;
@@ -58,28 +58,6 @@ server {
 	ssl_prefer_server_ciphers off;
 
 	location / {
-		proxy_pass http://localhost:8444/;
-	}
-}
-
-# Pokken, comment this out if you don't plan on serving pokken.
-server {
-	listen 443 ssl;	
-	server_name pokken.hostname.here;
-	
-	ssl_certificate /path/to/cert/pokken.pem;
-	ssl_certificate_key /path/to/cert/pokken.key;
-	ssl_session_timeout 1d;
-	ssl_session_cache shared:MozSSL:10m;
-	ssl_session_tickets off;
-
-	ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
-	ssl_ciphers "ALL:@SECLEVEL=0";
-	ssl_prefer_server_ciphers off;
-
-	location / {
-		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-		proxy_pass_request_headers on;
 		proxy_pass http://localhost:8080/;
 	}
 }
@@ -91,12 +69,12 @@ server {
 
     location / {
         return 301 https://$host$request_uri;
-		# If you don't want https redirection, comment the line above and uncomment the line below
+		# If you don't want https redirection, or are using something like cloudflare to manage HTTPS, comment out the line above and uncomment the line below
-		# proxy_pass http://localhost:8090/;
+		# proxy_pass http://localhost:8080/;
     }
 }
 
-# Frontend HTTPS. Comment out if you on't intend to use the frontend
+# Frontend HTTPS. Comment out if you on't intend to use the frontend, or have cloudflare or something managing https for you.
 server {
     listen 443 ssl;
 	server_name frontend.hostname.here;
@@ -118,6 +96,6 @@ server {
 	location / {
 		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 		proxy_pass_request_headers on;
-		proxy_pass http://localhost:8090/;
+		proxy_pass http://localhost:8080/;
 	}
 }

index.py

@@ -1,335 +1,116 @@
 #!/usr/bin/env python3
 import argparse
-import logging, coloredlogs
-from logging.handlers import TimedRotatingFileHandler
-from typing import Dict
 import yaml
-from os import path, mkdir, access, W_OK
+from os import path, environ
-from core import *
+import uvicorn
+import logging
+import asyncio
 
-from twisted.web import server, resource
+from core import CoreConfig, AimedbServlette
-from twisted.internet import reactor, endpoints
-from twisted.web.http import Request
-from routes import Mapper
-from threading import Thread
 
-class HttpDispatcher(resource.Resource):
+async def launch_main(cfg: CoreConfig, ssl: bool) -> None:
-    def __init__(self, cfg: CoreConfig, config_dir: str):
+    if ssl:
-        super().__init__()
+        server_cfg = uvicorn.Config(
-        self.config = cfg
+            "core.app:app", 
-        self.isLeaf = True
+            host=cfg.server.listen_address, 
-        self.map_get = Mapper()
+            port=cfg.server.port if args.port == 0 else args.port, 
-        self.map_post = Mapper()
+            reload=cfg.server.is_develop,
-        self.logger = logging.getLogger("core")
+            log_level="info" if cfg.server.is_develop else "critical",
-
+            ssl_version=3,
-        self.title = TitleServlet(cfg, config_dir)
+            ssl_certfile=cfg.server.ssl_cert,
-        self.allnet = AllnetServlet(cfg, config_dir)
+            ssl_keyfile=cfg.server.ssl_key
-        self.mucha = MuchaServlet(cfg, config_dir)
-
-        self.map_get.connect(
-            "allnet_downloadorder_ini",
-            "/dl/ini/{file}",
-            controller="allnet",
-            action="handle_dlorder_ini",
-            conditions=dict(method=["GET"]),
         ) 
-
-        self.map_post.connect(
-            "allnet_downloadorder_report",
-            "/report-api/Report",
-            controller="allnet",
-            action="handle_dlorder_report",
-            conditions=dict(method=["POST"]),
-        )
-
-        self.map_get.connect(
-            "allnet_ping",
-            "/naomitest.html",
-            controller="allnet",
-            action="handle_naomitest",
-            conditions=dict(method=["GET"]),
-        )
-        self.map_post.connect(
-            "allnet_poweron",
-            "/sys/servlet/PowerOn",
-            controller="allnet",
-            action="handle_poweron",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "allnet_downloadorder",
-            "/sys/servlet/DownloadOrder",
-            controller="allnet",
-            action="handle_dlorder",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "allnet_loaderstaterecorder",
-            "/sys/servlet/LoaderStateRecorder",
-            controller="allnet",
-            action="handle_loaderstaterecorder",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "allnet_alive",
-            "/sys/servlet/Alive",
-            controller="allnet",
-            action="handle_alive",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_get.connect(
-            "allnet_alive",
-            "/sys/servlet/Alive",
-            controller="allnet",
-            action="handle_alive",
-            conditions=dict(method=["GET"]),
-        )
-        self.map_post.connect(
-            "allnet_billing",
-            "/request",
-            controller="allnet",
-            action="handle_billing_request",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "allnet_billing",
-            "/request/",
-            controller="allnet",
-            action="handle_billing_request",
-            conditions=dict(method=["POST"]),
-        )
-
-        # Maintain compatability
-        self.map_post.connect(
-            "mucha_boardauth",
-            "/mucha/boardauth.do",
-            controller="mucha",
-            action="handle_boardauth",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "mucha_updatacheck",
-            "/mucha/updatacheck.do",
-            controller="mucha",
-            action="handle_updatecheck",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "mucha_dlstate",
-            "/mucha/downloadstate.do",
-            controller="mucha",
-            action="handle_dlstate",
-            conditions=dict(method=["POST"]),
-        )
-
-        self.map_post.connect(
-            "mucha_boardauth",
-            "/mucha_front/boardauth.do",
-            controller="mucha",
-            action="handle_boardauth",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "mucha_updatacheck",
-            "/mucha_front/updatacheck.do",
-            controller="mucha",
-            action="handle_updatecheck",
-            conditions=dict(method=["POST"]),
-        )
-        self.map_post.connect(
-            "mucha_dlstate",
-            "/mucha_front/downloadstate.do",
-            controller="mucha",
-            action="handle_dlstate",
-            conditions=dict(method=["POST"]),
-        )
-
-        for code, game in self.title.title_registry.items():
-            get_matchers, post_matchers = game.get_endpoint_matchers()
-            
-            for m in get_matchers:
-                self.map_get.connect(
-                    "title_get",
-                    m[1],
-                    controller="title",
-                    action="render_GET",
-                    title=code,
-                    subaction=m[0],
-                    conditions=dict(method=["GET"]),
-                    requirements=m[2],
-                )
-                
-            for m in post_matchers:
-                self.map_post.connect(
-                    "title_post",
-                    m[1],
-                    controller="title",
-                    action="render_POST",
-                    title=code,
-                    subaction=m[0],
-                    conditions=dict(method=["POST"]),
-                    requirements=m[2],
-                )
-
-    def render_GET(self, request: Request) -> bytes:
-        test = self.map_get.match(request.uri.decode())
-        client_ip = Utils.get_ip_addr(request)
-
-        if test is None:
-            self.logger.debug(
-                f"Unknown GET endpoint {request.uri.decode()} from {client_ip} to port {request.getHost().port}"
-            )
-            request.setResponseCode(404)
-            return b"Endpoint not found."
-
-        return self.dispatch(test, request)
-
-    def render_POST(self, request: Request) -> bytes:
-        test = self.map_post.match(request.uri.decode())
-        client_ip = Utils.get_ip_addr(request)
-
-        if test is None:
-            self.logger.debug(
-                f"Unknown POST endpoint {request.uri.decode()} from {client_ip} to port {request.getHost().port}"
-            )
-            request.setResponseCode(404)
-            return b"Endpoint not found."
-
-        return self.dispatch(test, request)
-
-    def dispatch(self, matcher: Dict, request: Request) -> bytes:
-        controller = getattr(self, matcher["controller"], None)
-        if controller is None:
-            self.logger.error(
-                f"Controller {matcher['controller']} not found via endpoint {request.uri.decode()}"
-            )
-            request.setResponseCode(404)
-            return b"Endpoint not found."
-
-        handler = getattr(controller, matcher["action"], None)
-        if handler is None:
-            self.logger.error(
-                f"Action {matcher['action']} not found in controller {matcher['controller']} via endpoint {request.uri.decode()}"
-            )
-            request.setResponseCode(404)
-            return b"Endpoint not found."
-
-        url_vars = matcher
-        url_vars.pop("controller")
-        url_vars.pop("action")
-        ret = handler(request, url_vars)
-
-        if type(ret) == str:
-            return ret.encode()
-        
-        elif type(ret) == bytes or type(ret) == tuple: # allow for bytes or tuple (data, response code) responses
-            return ret
-        
-        elif ret is None:
-            self.logger.warning(f"None returned by controller for {request.uri.decode()} endpoint")
-            return b""
-        
     else:
-            self.logger.warning(f"Unknown data type returned by controller for {request.uri.decode()} endpoint")
+        server_cfg = uvicorn.Config(
-            return b""
+            "core.app:app", 
+            host=cfg.server.listen_address, 
+            port=cfg.server.port if args.port == 0 else args.port, 
+            reload=cfg.server.is_develop,
+            log_level="info" if cfg.server.is_develop else "critical"
+        )
+    server = uvicorn.Server(server_cfg)
+    await server.serve()
 
+async def launch_billing(cfg: CoreConfig) -> None:
+    server_cfg = uvicorn.Config(
+        "core.allnet:app_billing", 
+        host=cfg.server.listen_address, 
+        port=cfg.billing.port, 
+        reload=cfg.server.is_develop,
+        log_level="info" if cfg.server.is_develop else "critical",
+        ssl_version=3,
+        ssl_certfile=cfg.billing.ssl_cert,
+        ssl_keyfile=cfg.billing.ssl_key
+    )
+    server = uvicorn.Server(server_cfg)
+    await server.serve()
+
+async def launch_frontend(cfg: CoreConfig) -> None:
+    server_cfg = uvicorn.Config(
+        "core.frontend:app", 
+        host=cfg.server.listen_address, 
+        port=cfg.frontend.port, 
+        reload=cfg.server.is_develop,
+        log_level="info" if cfg.server.is_develop else "critical",
+    )
+    server = uvicorn.Server(server_cfg)
+    await server.serve()
+
+async def launch_allnet(cfg: CoreConfig) -> None:
+    server_cfg = uvicorn.Config(
+        "core.allnet:app_allnet", 
+        host=cfg.server.listen_address, 
+        port=cfg.allnet.port, 
+        reload=cfg.server.is_develop,
+        log_level="info" if cfg.server.is_develop else "critical",
+    )
+    server = uvicorn.Server(server_cfg)
+    await server.serve()
+
+
+async def launcher(cfg: CoreConfig, ssl: bool) -> None:
+    task_list = [asyncio.create_task(launch_main(cfg, ssl))]
+    
+    if cfg.billing.standalone:
+        task_list.append(asyncio.create_task(launch_billing(cfg)))
+    if cfg.frontend.enable:
+        task_list.append(asyncio.create_task(launch_frontend(cfg)))
+    if cfg.allnet.standalone:
+        task_list.append(asyncio.create_task(launch_allnet(cfg)))
+    if cfg.aimedb.enable:
+        AimedbServlette(cfg).start()
+    
+    done, pending = await asyncio.wait(
+        task_list,
+        return_when=asyncio.FIRST_COMPLETED,
+    )
+    
+    logging.getLogger("core").info("Shutdown")
+    for pending_task in pending:
+        pending_task.cancel("Another service died, server is shutting down")
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="ARTEMiS main entry point")
+    parser = argparse.ArgumentParser(description="Artemis main entry point")
     parser.add_argument(
         "--config", "-c", type=str, default="config", help="Configuration folder"
     )
+    parser.add_argument(
+        "--port", "-p", type=int, default=0, help="Port override"
+    )
+    parser.add_argument(
+        "--ssl", "-s", type=bool, help="Launch with SSL"
+    )
     args = parser.parse_args()
 
     if not path.exists(f"{args.config}/core.yaml"):
         print(
-            f"The config folder you specified ({args.config}) does not exist or does not contain core.yaml.\nDid you copy the example folder?"
+            f"The config folder you specified ({args.config}) does not exist or does not contain core.yaml. Defaults will be used.\nDid you copy the example folder?"
         )
-        exit(1)
     
     cfg: CoreConfig = CoreConfig()
     if path.exists(f"{args.config}/core.yaml"):
         cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
 
-    if not path.exists(cfg.server.log_dir):
+    environ["ARTEMIS_CFG_DIR"] = args.config
-        mkdir(cfg.server.log_dir)
 
-    if not access(cfg.server.log_dir, W_OK):
+    asyncio.run(launcher(cfg, args.ssl))
-        print(
-            f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
-        )
-        exit(1)
-
-    logger = logging.getLogger("core")
-    log_fmt_str = "[%(asctime)s] Core | %(levelname)s | %(message)s"
-    log_fmt = logging.Formatter(log_fmt_str)
-
-    fileHandler = TimedRotatingFileHandler(
-        "{0}/{1}.log".format(cfg.server.log_dir, "core"), when="d", backupCount=10
-    )
-    fileHandler.setFormatter(log_fmt)
-
-    consoleHandler = logging.StreamHandler()
-    consoleHandler.setFormatter(log_fmt)
-
-    logger.addHandler(fileHandler)
-    logger.addHandler(consoleHandler)
-
-    log_lv = logging.DEBUG if cfg.server.is_develop else logging.INFO
-    logger.setLevel(log_lv)
-    coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
-
-    if not cfg.aimedb.key:
-        logger.error("!!AIMEDB KEY BLANK, SET KEY IN CORE.YAML!!")
-        exit(1)
-
-    logger.info(
-        f"ARTEMiS starting in {'develop' if cfg.server.is_develop else 'production'} mode"
-    )
-
-    allnet_server_str = f"tcp:{cfg.allnet.port}:interface={cfg.server.listen_address}"
-    title_server_str = f"tcp:{cfg.title.port}:interface={cfg.server.listen_address}"
-    title_https_server_str = f"ssl:{cfg.title.port_ssl}:interface={cfg.server.listen_address}:privateKey={cfg.title.ssl_key}:certKey={cfg.title.ssl_cert}"
-    adb_server_str = f"tcp:{cfg.aimedb.port}:interface={cfg.server.listen_address}"
-    frontend_server_str = (
-        f"tcp:{cfg.frontend.port}:interface={cfg.server.listen_address}"
-    )
-
-    billing_server_str = f"tcp:{cfg.billing.port}:interface={cfg.server.listen_address}"
-    if cfg.server.is_develop:
-        billing_server_str = (
-            f"ssl:{cfg.billing.port}:interface={cfg.server.listen_address}"
-            f":privateKey={cfg.billing.ssl_key}:certKey={cfg.billing.ssl_cert}"
-        )
-
-    dispatcher = HttpDispatcher(cfg, args.config)
-
-    endpoints.serverFromString(reactor, allnet_server_str).listen(
-        server.Site(dispatcher)
-    )
-    endpoints.serverFromString(reactor, adb_server_str).listen(AimedbFactory(cfg))
-
-    if cfg.frontend.enable:
-        endpoints.serverFromString(reactor, frontend_server_str).listen(
-            server.Site(FrontendServlet(cfg, args.config))
-        )
-
-    if cfg.billing.port > 0:
-        endpoints.serverFromString(reactor, billing_server_str).listen(
-            server.Site(dispatcher)
-        )
-
-    if cfg.title.port > 0:
-        endpoints.serverFromString(reactor, title_server_str).listen(
-            server.Site(dispatcher)
-        )
-    
-    if cfg.title.port_ssl > 0:
-        endpoints.serverFromString(reactor, title_https_server_str).listen(
-            server.Site(dispatcher)
-        )
-
-    if cfg.server.threading:
-        Thread(target=reactor.run, args=(False,)).start()
-    else:
-        reactor.run()

read.py

@@ -1,4 +1,4 @@
-# vim: set fileencoding=utf-8
+#!/usr/bin/env python3
 import argparse
 import re
 import os
@@ -6,6 +6,7 @@ import yaml
 from os import path
 import logging
 import coloredlogs
+import asyncio
 
 from logging.handlers import TimedRotatingFileHandler
 from typing import List, Optional
@@ -39,6 +40,9 @@ class BaseReader:
 
         return ret
     
+    async def read(self) -> None:
+        pass
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Import Game Information")
@@ -136,6 +140,8 @@ if __name__ == "__main__":
     for dir, mod in titles.items():
         if args.game in mod.game_codes:
             handler = mod.reader(config, args.version, bin_arg, opt_arg, args.extra)
-            handler.read()
+            loop = asyncio.get_event_loop()
+            loop.run_until_complete(handler.read())
+            
 
     logger.info("Done")

requirements.txt

@@ -1,6 +1,5 @@
 mypy
 wheel
-twisted
 pytz
 pyyaml
 sqlalchemy==1.4.46
@@ -12,10 +11,12 @@ inflection
 coloredlogs
 pylibmc; platform_system != "Windows"
 wacky
-Routes
 bcrypt
 jinja2
 protobuf
-autobahn
 pillow
-pyjwt
+pyjwt==2.8.0
+websockets
+starlette
+asyncio
+uvicorn

titles/chuni/__init__.py

@@ -7,4 +7,3 @@ index = ChuniServlet
 database = ChuniData
 reader = ChuniReader
 game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW, ChuniConstants.GAME_CODE_INT]
-current_schema_version = 5

titles/chuni/air.py

@@ -11,7 +11,7 @@ class ChuniAir(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AIR
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.10.00"
         return ret

titles/chuni/airplus.py

@@ -11,7 +11,7 @@ class ChuniAirPlus(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AIR_PLUS
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.15.00"
         return ret

titles/chuni/amazon.py

@@ -13,7 +13,7 @@ class ChuniAmazon(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AMAZON
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.30.00"
         return ret

titles/chuni/amazonplus.py

@@ -13,7 +13,7 @@ class ChuniAmazonPlus(ChuniBase):
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AMAZON_PLUS
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_setting_api_request(data)
+        ret = await super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.35.00"
         return ret

titles/chuni/base.py

@@ -22,7 +22,7 @@ class ChuniBase:
         self.game = ChuniConstants.GAME_CODE
         self.version = ChuniConstants.VER_CHUNITHM
 
-    def handle_game_login_api_request(self, data: Dict) -> Dict:
+    async def handle_game_login_api_request(self, data: Dict) -> Dict:
         """
         Handles the login bonus logic, required for the game because
         getUserLoginBonus gets called after getUserItem and therefore the
@@ -38,20 +38,20 @@ class ChuniBase:
             return {"returnCode": 1}
 
         user_id = data["userId"]
-        login_bonus_presets = self.data.static.get_login_bonus_presets(self.version)
+        login_bonus_presets = await self.data.static.get_login_bonus_presets(self.version)
 
         for preset in login_bonus_presets:
             # check if a user already has some pogress and if not add the
             # login bonus entry
-            user_login_bonus = self.data.item.get_login_bonus(
+            user_login_bonus = await self.data.item.get_login_bonus(
                 user_id, self.version, preset["presetId"]
             )
             if user_login_bonus is None:
-                self.data.item.put_login_bonus(
+                await self.data.item.put_login_bonus(
                     user_id, self.version, preset["presetId"]
                 )
                 # yeah i'm lazy
-                user_login_bonus = self.data.item.get_login_bonus(
+                user_login_bonus = await self.data.item.get_login_bonus(
                     user_id, self.version, preset["presetId"]
                 )
 
@@ -67,7 +67,7 @@ class ChuniBase:
                 bonus_count = user_login_bonus["bonusCount"] + 1
                 last_update_date = datetime.now()
 
-                all_login_boni = self.data.static.get_login_bonus(
+                all_login_boni = await self.data.static.get_login_bonus(
                     self.version, preset["presetId"]
                 )
 
@@ -91,13 +91,13 @@ class ChuniBase:
                         is_finished = True
 
                 # grab the item for the corresponding day
-                login_item = self.data.static.get_login_bonus_by_required_days(
+                login_item = await self.data.static.get_login_bonus_by_required_days(
                     self.version, preset["presetId"], bonus_count
                 )
                 if login_item is not None:
                     # now add the present to the database so the
                     # handle_get_user_item_api_request can grab them
-                    self.data.item.put_item(
+                    await self.data.item.put_item(
                         user_id,
                         {
                             "itemId": login_item["presentId"],
@@ -107,7 +107,7 @@ class ChuniBase:
                         },
                     )
 
-                self.data.item.put_login_bonus(
+                await self.data.item.put_login_bonus(
                     user_id,
                     self.version,
                     preset["presetId"],
@@ -119,12 +119,12 @@ class ChuniBase:
 
         return {"returnCode": 1}
 
-    def handle_game_logout_api_request(self, data: Dict) -> Dict:
+    async def handle_game_logout_api_request(self, data: Dict) -> Dict:
         # self.data.base.log_event("chuni", "logout", logging.INFO, {"version": self.version, "user": data["userId"]})
         return {"returnCode": 1}
 
-    def handle_get_game_charge_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_charge_api_request(self, data: Dict) -> Dict:
-        game_charge_list = self.data.static.get_enabled_charges(self.version)
+        game_charge_list = await self.data.static.get_enabled_charges(self.version)
 
         if game_charge_list is None or len(game_charge_list) == 0:
             return {"length": 0, "gameChargeList": []}
@@ -145,8 +145,8 @@ class ChuniBase:
             )
         return {"length": len(charges), "gameChargeList": charges}
 
-    def handle_get_game_event_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_event_api_request(self, data: Dict) -> Dict:
-        game_events = self.data.static.get_enabled_events(self.version)
+        game_events = await self.data.static.get_enabled_events(self.version)
 
         if game_events is None or len(game_events) == 0:
             self.logger.warning("No enabled events, did you run the reader?")
@@ -177,10 +177,10 @@ class ChuniBase:
             "gameEventList": event_list,
         }
 
-    def handle_get_game_idlist_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_idlist_api_request(self, data: Dict) -> Dict:
         return {"type": data["type"], "length": 0, "gameIdlistList": []}
 
-    def handle_get_game_message_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_message_api_request(self, data: Dict) -> Dict:
         return {
             "type": data["type"], 
             "length": 1, 
@@ -193,14 +193,14 @@ class ChuniBase:
             }]
         }
 
-    def handle_get_game_ranking_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_ranking_api_request(self, data: Dict) -> Dict:
-        rankings = self.data.score.get_rankings(self.version)
+        rankings = await self.data.score.get_rankings(self.version)
         return {"type": data["type"], "gameRankingList": rankings}
 
-    def handle_get_game_sale_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_sale_api_request(self, data: Dict) -> Dict:
         return {"type": data["type"], "length": 0, "gameSaleList": []}
 
-    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         # if reboot start/end time is not defined use the default behavior of being a few hours ago
         if self.core_cfg.title.reboot_start_time == "" or self.core_cfg.title.reboot_end_time == "":
             reboot_start = datetime.strftime(
@@ -240,8 +240,8 @@ class ChuniBase:
             "isDumpUpload": "false",
             "isAou": "false",
         }
-    def handle_get_user_activity_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_activity_api_request(self, data: Dict) -> Dict:
-        user_activity_list = self.data.profile.get_profile_activity(
+        user_activity_list = await self.data.profile.get_profile_activity(
             data["userId"], data["kind"]
         )
 
@@ -261,8 +261,8 @@ class ChuniBase:
             "userActivityList": activity_list,
         }
 
-    def handle_get_user_character_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_character_api_request(self, data: Dict) -> Dict:
-        characters = self.data.item.get_characters(data["userId"])
+        characters = await self.data.item.get_characters(data["userId"])
         if characters is None:
             return {
                 "userId": data["userId"],
@@ -296,8 +296,8 @@ class ChuniBase:
             "userCharacterList": character_list,
         }
 
-    def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
-        user_charge_list = self.data.profile.get_profile_charge(data["userId"])
+        user_charge_list = await self.data.profile.get_profile_charge(data["userId"])
 
         charge_list = []
         for charge in user_charge_list:
@@ -312,15 +312,15 @@ class ChuniBase:
             "userChargeList": charge_list,
         }
 
-    def handle_get_user_recent_player_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_recent_player_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 0,
             "userRecentPlayerList": [], # playUserId, playUserName, playDate, friendPoint
         }
 
-    def handle_get_user_course_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_course_api_request(self, data: Dict) -> Dict:
-        user_course_list = self.data.score.get_courses(data["userId"])
+        user_course_list = await self.data.score.get_courses(data["userId"])
         if user_course_list is None:
             return {
                 "userId": data["userId"],
@@ -354,8 +354,8 @@ class ChuniBase:
             "userCourseList": course_list,
         }
 
-    def handle_get_user_data_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_data_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_data(data["userId"], self.version)
+        p = await self.data.profile.get_profile_data(data["userId"], self.version)
         if p is None:
             return {}
 
@@ -366,8 +366,8 @@ class ChuniBase:
 
         return {"userId": data["userId"], "userData": profile}
 
-    def handle_get_user_data_ex_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_data_ex_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_data_ex(data["userId"], self.version)
+        p = await self.data.profile.get_profile_data_ex(data["userId"], self.version)
         if p is None:
             return {}
 
@@ -378,8 +378,8 @@ class ChuniBase:
 
         return {"userId": data["userId"], "userDataEx": profile}
 
-    def handle_get_user_duel_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_duel_api_request(self, data: Dict) -> Dict:
-        user_duel_list = self.data.item.get_duels(data["userId"])
+        user_duel_list = await self.data.item.get_duels(data["userId"])
         if user_duel_list is None:
             return {}
 
@@ -396,8 +396,8 @@ class ChuniBase:
             "userDuelList": duel_list,
         }
 
-    def handle_get_user_rival_data_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_rival_data_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_rival(data["rivalId"])
+        p = await self.data.profile.get_rival(data["rivalId"])
         if p is None:
             return {}
         userRivalData = {
@@ -409,14 +409,14 @@ class ChuniBase:
             "userRivalData": userRivalData
         }
     
-    def handle_get_user_rival_music_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_rival_music_api_request(self, data: Dict) -> Dict:
         rival_id = data["rivalId"]
         next_index = int(data["nextIndex"])
         max_count = int(data["maxCount"])
         user_rival_music_list = []
 
         # Fetch all the rival music entries for the user
-        all_entries = self.data.score.get_rival_music(rival_id)
+        all_entries = await self.data.score.get_rival_music(rival_id)
 
         # Process the entries based on max_count and nextIndex
         for music in all_entries:
@@ -462,12 +462,12 @@ class ChuniBase:
         return result
 
     
-    def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
         user_fav_item_list = []
 
         # still needs to be implemented on WebUI
         # 1: Music, 2: User, 3: Character
-        fav_list = self.data.item.get_all_favorites(
+        fav_list = await self.data.item.get_all_favorites(
             data["userId"], self.version, fav_kind=int(data["kind"])
         )
         if fav_list is not None:
@@ -482,17 +482,17 @@ class ChuniBase:
             "userFavoriteItemList": user_fav_item_list,
         }
 
-    def handle_get_user_favorite_music_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_favorite_music_api_request(self, data: Dict) -> Dict:
         """
         This is handled via the webui, which we don't have right now
         """
 
         return {"userId": data["userId"], "length": 0, "userFavoriteMusicList": []}
 
-    def handle_get_user_item_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_item_api_request(self, data: Dict) -> Dict:
         kind = int(int(data["nextIndex"]) / 10000000000)
         next_idx = int(int(data["nextIndex"]) % 10000000000)
-        user_item_list = self.data.item.get_items(data["userId"], kind)
+        user_item_list = await self.data.item.get_items(data["userId"], kind)
 
         if user_item_list is None or len(user_item_list) == 0:
             return {
@@ -526,9 +526,9 @@ class ChuniBase:
             "userItemList": items,
         }
 
-    def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
         user_id = data["userId"]
-        user_login_bonus = self.data.item.get_all_login_bonus(user_id, self.version)
+        user_login_bonus = await self.data.item.get_all_login_bonus(user_id, self.version)
         # ignore the loginBonus request if its disabled in config
         if user_login_bonus is None or not self.game_cfg.mods.use_login_bonus:
             return {"userId": user_id, "length": 0, "userLoginBonusList": []}
@@ -552,8 +552,8 @@ class ChuniBase:
             "userLoginBonusList": user_login_list,
         }
 
-    def handle_get_user_map_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_map_api_request(self, data: Dict) -> Dict:
-        user_map_list = self.data.item.get_maps(data["userId"])
+        user_map_list = await self.data.item.get_maps(data["userId"])
         if user_map_list is None:
             return {}
 
@@ -570,8 +570,8 @@ class ChuniBase:
             "userMapList": map_list,
         }
 
-    def handle_get_user_music_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_music_api_request(self, data: Dict) -> Dict:
-        music_detail = self.data.score.get_scores(data["userId"])
+        music_detail = await self.data.score.get_scores(data["userId"])
         if music_detail is None:
             return {
                 "userId": data["userId"],
@@ -629,8 +629,8 @@ class ChuniBase:
             "userMusicList": song_list,  # 240
         }
 
-    def handle_get_user_option_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_option_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_option(data["userId"])
+        p = await self.data.profile.get_profile_option(data["userId"])
 
         option = p._asdict()
         option.pop("id")
@@ -638,8 +638,8 @@ class ChuniBase:
 
         return {"userId": data["userId"], "userGameOption": option}
 
-    def handle_get_user_option_ex_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_option_ex_api_request(self, data: Dict) -> Dict:
-        p = self.data.profile.get_profile_option_ex(data["userId"])
+        p = await self.data.profile.get_profile_option_ex(data["userId"])
 
         option = p._asdict()
         option.pop("id")
@@ -650,11 +650,11 @@ class ChuniBase:
     def read_wtf8(self, src):
         return bytes([ord(c) for c in src]).decode("utf-8")
 
-    def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
-        profile = self.data.profile.get_profile_preview(data["userId"], self.version)
+        profile = await self.data.profile.get_profile_preview(data["userId"], self.version)
         if profile is None:
             return None
-        profile_character = self.data.item.get_character(
+        profile_character = await self.data.item.get_character(
             data["userId"], profile["characterId"]
         )
 
@@ -692,8 +692,8 @@ class ChuniBase:
             "userNameEx": profile["userName"],
         }
 
-    def handle_get_user_recent_rating_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_recent_rating_api_request(self, data: Dict) -> Dict:
-        recent_rating_list = self.data.profile.get_profile_recent_rating(data["userId"])
+        recent_rating_list = await self.data.profile.get_profile_recent_rating(data["userId"])
         if recent_rating_list is None:
             return {
                 "userId": data["userId"],
@@ -707,7 +707,7 @@ class ChuniBase:
             "userRecentRatingList": recent_rating_list["recentRating"],
         }
 
-    def handle_get_user_region_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_region_api_request(self, data: Dict) -> Dict:
         # TODO: Region
         return {
             "userId": data["userId"],
@@ -715,22 +715,22 @@ class ChuniBase:
             "userRegionList": [],
         }
 
-    def handle_get_user_team_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_team_api_request(self, data: Dict) -> Dict:
         # Default values
         team_id = 65535
         team_name = self.game_cfg.team.team_name
         team_rank = 0
 
         # Get user profile
-        profile = self.data.profile.get_profile_data(data["userId"], self.version)
+        profile = await self.data.profile.get_profile_data(data["userId"], self.version)
         if profile and profile["teamId"]:
             # Get team by id
-            team = self.data.profile.get_team_by_id(profile["teamId"])
+            team = await self.data.profile.get_team_by_id(profile["teamId"])
 
             if team:
                 team_id = team["id"]
                 team_name = team["teamName"]
-                team_rank = self.data.profile.get_team_rank(team["id"])
+                team_rank = await self.data.profile.get_team_rank(team["id"])
 
         # Don't return anything if no team name has been defined for defaults and there is no team set for the player
         if not profile["teamId"] and team_name == "":
@@ -750,7 +750,7 @@ class ChuniBase:
             },
         }
     
-    def handle_get_team_course_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_get_team_course_setting_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 0,
@@ -758,7 +758,7 @@ class ChuniBase:
             "teamCourseSettingList": [],
         }
 
-    def handle_get_team_course_setting_api_request_proto(self, data: Dict) -> Dict:
+    async def handle_get_team_course_setting_api_request_proto(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 1,
@@ -782,7 +782,7 @@ class ChuniBase:
             ],
         }
 
-    def handle_get_team_course_rule_api_request(self, data: Dict) -> Dict:
+    async def handle_get_team_course_rule_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 0,
@@ -790,7 +790,7 @@ class ChuniBase:
             "teamCourseRuleList": []
         }
 
-    def handle_get_team_course_rule_api_request_proto(self, data: Dict) -> Dict:
+    async def handle_get_team_course_rule_api_request_proto(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "length": 1,
@@ -807,7 +807,7 @@ class ChuniBase:
             ],
         }
 
-    def handle_upsert_user_all_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_user_all_api_request(self, data: Dict) -> Dict:
         upsert = data["upsertUserAll"]
         user_id = data["userId"]
 
@@ -819,58 +819,58 @@ class ChuniBase:
             except Exception:
                 pass
 
-            self.data.profile.put_profile_data(
+            await self.data.profile.put_profile_data(
                 user_id, self.version, upsert["userData"][0]
             )
 
         if "userDataEx" in upsert:
-            self.data.profile.put_profile_data_ex(
+            await self.data.profile.put_profile_data_ex(
                 user_id, self.version, upsert["userDataEx"][0]
             )
 
         if "userGameOption" in upsert:
-            self.data.profile.put_profile_option(user_id, upsert["userGameOption"][0])
+            await self.data.profile.put_profile_option(user_id, upsert["userGameOption"][0])
 
         if "userGameOptionEx" in upsert:
-            self.data.profile.put_profile_option_ex(
+            await self.data.profile.put_profile_option_ex(
                 user_id, upsert["userGameOptionEx"][0]
             )
         if "userRecentRatingList" in upsert:
-            self.data.profile.put_profile_recent_rating(
+            await self.data.profile.put_profile_recent_rating(
                 user_id, upsert["userRecentRatingList"]
             )
 
         if "userCharacterList" in upsert:
             for character in upsert["userCharacterList"]:
-                self.data.item.put_character(user_id, character)
+                await self.data.item.put_character(user_id, character)
 
         if "userMapList" in upsert:
             for map in upsert["userMapList"]:
-                self.data.item.put_map(user_id, map)
+                await self.data.item.put_map(user_id, map)
 
         if "userCourseList" in upsert:
             for course in upsert["userCourseList"]:
-                self.data.score.put_course(user_id, course)
+                await self.data.score.put_course(user_id, course)
 
         if "userDuelList" in upsert:
             for duel in upsert["userDuelList"]:
-                self.data.item.put_duel(user_id, duel)
+                await self.data.item.put_duel(user_id, duel)
 
         if "userItemList" in upsert:
             for item in upsert["userItemList"]:
-                self.data.item.put_item(user_id, item)
+                await self.data.item.put_item(user_id, item)
 
         if "userActivityList" in upsert:
             for activity in upsert["userActivityList"]:
-                self.data.profile.put_profile_activity(user_id, activity)
+                await self.data.profile.put_profile_activity(user_id, activity)
 
         if "userChargeList" in upsert:
             for charge in upsert["userChargeList"]:
-                self.data.profile.put_profile_charge(user_id, charge)
+                await self.data.profile.put_profile_charge(user_id, charge)
 
         if "userMusicDetailList" in upsert:
             for song in upsert["userMusicDetailList"]:
-                self.data.score.put_score(user_id, song)
+                await self.data.score.put_score(user_id, song)
 
         if "userPlaylogList" in upsert:
             for playlog in upsert["userPlaylogList"]:
@@ -881,7 +881,7 @@ class ChuniBase:
                   playlog["playedUserName2"] = self.read_wtf8(playlog["playedUserName2"])
                 if playlog["playedUserName3"] is not None:
                   playlog["playedUserName3"] = self.read_wtf8(playlog["playedUserName3"])
-                self.data.score.put_playlog(user_id, playlog, self.version)
+                await self.data.score.put_playlog(user_id, playlog, self.version)
 
         if "userTeamPoint" in upsert:
             team_points = upsert["userTeamPoint"]
@@ -889,7 +889,7 @@ class ChuniBase:
                 for tp in team_points:
                     if tp["teamId"] != '65535':
                         # Fetch the current team data
-                        current_team = self.data.profile.get_team_by_id(tp["teamId"])
+                        current_team = await self.data.profile.get_team_by_id(tp["teamId"])
 
                         # Calculate the new teamPoint
                         new_team_point = int(tp["teamPoint"]) + current_team["teamPoint"]
@@ -900,24 +900,24 @@ class ChuniBase:
                         }
 
                         # Update the team data
-                        self.data.profile.update_team(tp["teamId"], team_data)
+                        await self.data.profile.update_team(tp["teamId"], team_data)
             except:
                 pass # Probably a better way to catch if the team is not set yet (new profiles), but let's just pass
         if "userMapAreaList" in upsert:
             for map_area in upsert["userMapAreaList"]:
-                self.data.item.put_map_area(user_id, map_area)
+                await self.data.item.put_map_area(user_id, map_area)
 
         if "userOverPowerList" in upsert:
             for overpower in upsert["userOverPowerList"]:
-                self.data.profile.put_profile_overpower(user_id, overpower)
+                await self.data.profile.put_profile_overpower(user_id, overpower)
 
         if "userEmoneyList" in upsert:
             for emoney in upsert["userEmoneyList"]:
-                self.data.profile.put_profile_emoney(user_id, emoney)
+                await self.data.profile.put_profile_emoney(user_id, emoney)
 
         if "userLoginBonusList" in upsert:
             for login in upsert["userLoginBonusList"]:
-                self.data.item.put_login_bonus(
+                await self.data.item.put_login_bonus(
                     user_id, self.version, login["presetId"], isWatched=True
                 )
         
@@ -927,28 +927,28 @@ class ChuniBase:
 
         return {"returnCode": "1"}
 
-    def handle_upsert_user_chargelog_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_user_chargelog_api_request(self, data: Dict) -> Dict:
         # add tickets after they got bought, this makes sure the tickets are
         # still valid after an unsuccessful logout
-        self.data.profile.put_profile_charge(data["userId"], data["userCharge"])
+        await self.data.profile.put_profile_charge(data["userId"], data["userCharge"])
         return {"returnCode": "1"}
 
-    def handle_upsert_client_bookkeeping_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_bookkeeping_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_upsert_client_develop_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_develop_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_upsert_client_error_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_error_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_upsert_client_setting_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_setting_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_upsert_client_testmode_api_request(self, data: Dict) -> Dict:
+    async def handle_upsert_client_testmode_api_request(self, data: Dict) -> Dict:
         return {"returnCode": "1"}
 
-    def handle_get_user_net_battle_data_api_request(self, data: Dict) -> Dict:
+    async def handle_get_user_net_battle_data_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
             "userNetBattleData": {"recentNBSelectMusicList": []},