Merge branch 'develop' into idac

Dockerfile

@@ -15,7 +15,6 @@ COPY dbutils.py dbutils.py
 COPY read.py read.py
 ADD core core
 ADD titles titles
-ADD config config
 ADD logs logs
 ADD cert cert
 

LICENSE.txt

@@ -0,0 +1,13 @@
+           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+                   Version 2, December 2004
+ 
+Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
+
+Everyone is permitted to copy and distribute verbatim or modified
+copies of this license document, and changing it is allowed as long
+as the name is changed.
+ 
+           DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
+  TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. You just DO WHAT THE FUCK YOU WANT TO.

changelog.md

@@ -1,14 +1,70 @@
 # Changelog
 Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to.
 
-## 20231015
+## 20240302
+### SAO
++ Fixing new profile creation with right heroes and start VP
++ Fix to the Unanalyzed Log responses returning the wrong rewards
++ Documentation revised
+
+## 20240226
+### CXB
++ Fixing paths for rev.py
++ Changed encoding for handle_data_item_list_icon_request
+
+## 20240202
+### SAO
++ Added reader assets and edited the game specific documentation
+
+## 20240118
+### System
++ Added game version names to the readme
+
+## 20240109
+### System
++ Removed `ADD config config` from dockerfile [#83](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/83) (Thanks zaphkito!)
+
+### Aimedb
++ Fixed an error that resulted from trying to scan a banned or locked card
+
+## 20240108
+### System
++ Change how the underlying system handles URLs
+  + This can now allow for things like version-specific, or even keychip-specific URLs
+  + Specific changes to games are noted below
++ Fix docker files [#60](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/60) (Thanks Rylie!)
++ Fix support for python 3.8 - 3.10
+
+### Aimedb
++ Add support for SegaAuth key in games that support it (for now only Chunithm)
+  + This is a JWT that is sent to games, by Aimedb, that the games send to their game server, to verify that the access code the game is sending to the server was obtained via aimedb.
+  + Requires a base64-encoded secret to be set in the `core.yaml`
+
+### Chunithm
++ Fix Air support
++ Add saving for userRecentPlayerList
++ Add support for SegaAuthKey
++ Fix a bug arising if a user set their name to be 'true' or 'false'
++ Add support for Sun+ [#78](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/78) (Thanks EmmyHeart!)
++ Add `matching` section to `chuni.yaml`
++ ~~Change `udpHolePunchUri` and `reflectorUri` to be STUN and TURN servers~~ Reverted
++ Imrpove `GetGameSetting` request handling for different versions
++ Fix issue where songs would not always return all scores [#92](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/92) (Thanks Kumubou!)
+
 ### maimai DX
-+ Added support for FESTiVAL PLUS
++ Fix user charges failing to save
 
-### Card Maker
+### maimai
-+ Added support for maimai DX FESTiVAL PLUS
++ Made it functional
+
+### CXB
++ Improvements to request dispatching
++ Add support for non-omnimix music lists
+
+
+### IDZ
++ Fix news urls in accordance with the system change to URLs
 
-## 20231001
 ### Initial D THE ARCADE
 + Added support for Initial D THE ARCADE S2
   + Story mode progress added
@@ -20,6 +76,45 @@ Documenting updates to ARTEMiS, to be updated every time the master branch is pu
   + Frontend to download profile added
   + Importer to import profiles added
 
+### ONGEKI
++ Now supports HTTPS on a per-version basis
++ Merg PR [#61](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/61) (Thanks phantomlan!)
+  + Add Ranking Event Support
+  + Add reward list support
+  + Add version segregation to Event Ranking, Tech Challenge, and Music Ranking
+  + Now stores ClientTestmode and ClientSetting data
++ Fix mission points not adding correctly [#68](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/68) (Thanks phantomlan!)
++ Fix tech challenge [#70](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/70) (Thanks phantomlan!)
+
+### SAO
++ Change endpoint in accordance with the system change to URLs
++ Update request header class to be more accurate
++ Encrypted requests are now supported
++ Change to using handler classes instead of raw structs for simplicity
+
+### Wacca
++ Fix a server error causing a seperate error that casued issues
++ Add better error printing
++ Add better request validation
++ Fix HousingStartV2
++ Fix Lily's housing/get handler
+
+## 20231107
+### CXB
++  Hotfix `render_POST` sometimes failing to read the request body on large requests
+
+## 20231106
+### CXB
++ Hotfix `render_POST` function signature signature
++ Hotfix `handle_action_addenergy_request` hard failing if `get_energy` returns None
+
+## 20231015
+### maimai DX
++ Added support for FESTiVAL PLUS
+
+### Card Maker
++ Added support for maimai DX FESTiVAL PLUS
+
 ## 20230716
 ### General
 + Docker files added (#19)

core/__init__.py

@@ -1,6 +1,6 @@
 from core.config import CoreConfig
-from core.allnet import AllnetServlet
+from core.allnet import AllnetServlet, BillingServlet
-from core.aimedb import AimedbFactory
+from core.aimedb import AimedbServlette
 from core.title import TitleServlet
 from core.utils import Utils
 from core.mucha import MuchaServlet

core/adb_handlers/base.py

@@ -102,7 +102,7 @@ class ADBHeader:
         magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id = struct.unpack_from("<5H6sI12s", data)
         head = cls(magic, protocol_ver, cmd, length, status, game_id, store_id, keychip_id)
         
-        if head.length != len(data):
+        if head.length > len(data):
             raise ADBHeaderException(f"Length is incorrect! Expect {head.length}, got {len(data)}")
         
         return head

core/aimedb.py

@@ -1,9 +1,7 @@
-from twisted.internet.protocol import Factory, Protocol
 import logging, coloredlogs
 from Crypto.Cipher import AES
-import struct
+from typing import Dict, Tuple, Callable, Union, Optional
-from typing import Dict, Tuple, Callable, Union
+import asyncio
-from typing_extensions import Final
 from logging.handlers import TimedRotatingFileHandler
 
 from core.config import CoreConfig
@@ -11,15 +9,37 @@ from core.utils import create_sega_auth_key
 from core.data import Data
 from .adb_handlers import *
 
-
+class AimedbServlette():
-class AimedbProtocol(Protocol):
     request_list: Dict[int, Tuple[Callable[[bytes, int], Union[ADBBaseResponse, bytes]], int, str]] = {}
-
     def __init__(self, core_cfg: CoreConfig) -> None:        
-        self.logger = logging.getLogger("aimedb")
         self.config = core_cfg        
         self.data = Data(core_cfg)
-        if core_cfg.aimedb.key == "":
+
+        self.logger = logging.getLogger("aimedb")
+        if not hasattr(self.logger, "initted"):
+            log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "aimedb"),
+                when="d",
+                backupCount=10,
+            )
+            fileHandler.setFormatter(log_fmt)
+
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
+
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(self.config.aimedb.loglevel)
+            coloredlogs.install(
+                level=core_cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.initted = True
+
+        if not core_cfg.aimedb.key:
             self.logger.error("!!!KEY NOT SET!!!")
             exit(1)
 
@@ -40,27 +60,31 @@ class AimedbProtocol(Protocol):
 
         self.register_handler(0x13, 0x14, self.handle_log_ex, 'aime_log_ex')
         self.register_handler(0x64, 0x65, self.handle_hello, 'hello')
-        self.register_handler(0x66, 0, self.handle_goodbye, 'goodbye')
 
     def register_handler(self, cmd: int, resp:int, handler: Callable[[bytes, int], Union[ADBBaseResponse, bytes]], name: str) -> None:
         self.request_list[cmd] = (handler, resp, name)
     
-    def append_padding(self, data: bytes):
+    def start(self) -> None:
-        """Appends 0s to the end of the data until it's at the correct size"""
+        self.logger.info(f"Start on port {self.config.aimedb.port}")
-        length = struct.unpack_from("<H", data, 6)
+        addr = self.config.aimedb.listen_address if self.config.aimedb.listen_address else self.config.server.listen_address
-        padding_size = length[0] - len(data)
+        asyncio.create_task(asyncio.start_server(self.dataReceived, addr, self.config.aimedb.port))
-        data += bytes(padding_size)
-        return data
     
-    def connectionMade(self) -> None:
+    async def dataReceived(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
-        self.logger.debug(f"{self.transport.getPeer().host} Connected")
+        self.logger.debug(f"Connection made from {writer.get_extra_info('peername')[0]}")
+        while True:
+            try:
+                data: bytes = await reader.read(4096)
+                if len(data) == 0:
+                    self.logger.debug("Connection closed")
+                    return
+                await self.process_data(data, reader, writer)
+                await writer.drain()
+            except ConnectionResetError as e:
+                self.logger.debug("Connection reset, disconnecting")
+                return
 
-    def connectionLost(self, reason) -> None:
+    async def process_data(self, data: bytes, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> Optional[bytes]:
-        self.logger.debug(
+        addr = writer.get_extra_info('peername')[0]
-            f"{self.transport.getPeer().host} Disconnected - {reason.value}"
-        )
-
-    def dataReceived(self, data: bytes) -> None:
         cipher = AES.new(self.config.aimedb.key.encode(), AES.MODE_ECB)
 
         try:
@@ -68,9 +92,9 @@ class AimedbProtocol(Protocol):
         
         except Exception as e:
             self.logger.error(f"Failed to decrypt {data.hex()} because {e}")
-            return None
+            return
 
-        self.logger.debug(f"{self.transport.getPeer().host} wrote {decrypted.hex()}")
+        self.logger.debug(f"{addr} wrote {decrypted.hex()}")
 
         try:
             head = ADBHeader.from_data(decrypted)
@@ -79,7 +103,9 @@ class AimedbProtocol(Protocol):
             self.logger.error(f"Error parsing ADB header: {e}")
             try:    
                 encrypted = cipher.encrypt(ADBBaseResponse().make())
-                self.transport.write(encrypted)
+                writer.write(encrypted)
+                await writer.drain()
+                return
 
             except Exception as e:
                 self.logger.error(f"Failed to encrypt default response because {e}")
@@ -89,46 +115,51 @@ class AimedbProtocol(Protocol):
         if head.keychip_id == "ABCD1234567" or head.store_id == 0xfff0:
             self.logger.warning(f"Request from uninitialized AMLib: {vars(head)}")
 
+        if head.cmd == 0x66:
+            self.logger.info("Goodbye")
+            writer.close()
+            return
+
         handler, resp_code, name = self.request_list.get(head.cmd, (self.handle_default, None, 'default'))
 
         if resp_code is None:
             self.logger.warning(f"No handler for cmd {hex(head.cmd)}")
         
         elif resp_code > 0:
-            self.logger.info(f"{name} from {head.keychip_id} ({head.game_id}) @ {self.transport.getPeer().host}")
+            self.logger.info(f"{name} from {head.keychip_id} ({head.game_id}) @ {addr}")
         
-        resp = handler(decrypted, resp_code)
+        resp = await handler(decrypted, resp_code)
 
         if type(resp) == ADBBaseResponse or issubclass(type(resp), ADBBaseResponse):
             resp_bytes = resp.make()
-            if len(resp_bytes) != resp.head.length:
-                resp_bytes = self.append_padding(resp_bytes)
 
         elif type(resp) == bytes:
             resp_bytes = resp
         
         elif resp is None: # Nothing to send, probably a goodbye
+            self.logger.warn(f"None return by handler for {name}")
             return
         
         else:
+            self.logger.error(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
             raise TypeError(f"Unsupported type returned by ADB handler for {name}: {type(resp)}")
 
         try:
             encrypted = cipher.encrypt(resp_bytes)
             self.logger.debug(f"Response {resp_bytes.hex()}")
-            self.transport.write(encrypted)
+            writer.write(encrypted)
 
         except Exception as e:
             self.logger.error(f"Failed to encrypt {resp_bytes.hex()} because {e}")
     
-    def handle_default(self, data: bytes, resp_code: int, length: int = 0x20) -> ADBBaseResponse:
+    async def handle_default(self, data: bytes, resp_code: int, length: int = 0x20) -> ADBBaseResponse:
         req = ADBHeader.from_data(data)
         return ADBBaseResponse(resp_code, length, 1, req.game_id, req.store_id, req.keychip_id, req.protocol_ver)
 
-    def handle_hello(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_hello(self, data: bytes, resp_code: int) -> ADBBaseResponse:
-        return self.handle_default(data, resp_code)
+        return await self.handle_default(data, resp_code)
 
-    def handle_campaign(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_campaign(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         h = ADBHeader.from_data(data)
         if h.protocol_ver >= 0x3030:
             req = h
@@ -143,31 +174,34 @@ class AimedbProtocol(Protocol):
         # We don't currently support campaigns
         return resp
 
-    def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_lookup(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         req = ADBLookupRequest(data)
-        user_id = self.data.card.get_user_id_from_card(req.access_code)
+        user_id = await self.data.card.get_user_id_from_card(req.access_code)
-        is_banned = self.data.card.get_card_banned(req.access_code)
+        is_banned = await self.data.card.get_card_banned(req.access_code)
-        is_locked = self.data.card.get_card_locked(req.access_code)
+        is_locked = await self.data.card.get_card_locked(req.access_code)
         
+        ret = ADBLookupResponse.from_req(req.head, user_id)
         if is_banned and is_locked:
             ret.head.status = ADBStatus.BAN_SYS_USER
         elif is_banned:
             ret.head.status = ADBStatus.BAN_SYS
         elif is_locked:
             ret.head.status = ADBStatus.LOCK_USER
-        ret = ADBLookupResponse.from_req(req.head, user_id)
         
         self.logger.info(
             f"access_code {req.access_code} -> user_id {ret.user_id}"
         )
+        
+        if user_id and user_id > 0:
+            await self.data.card.update_card_last_login(req.access_code)
         return ret
 
-    def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_lookup_ex(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         req = ADBLookupRequest(data)
-        user_id = self.data.card.get_user_id_from_card(req.access_code)
+        user_id = await self.data.card.get_user_id_from_card(req.access_code)
 
-        is_banned = self.data.card.get_card_banned(req.access_code)
+        is_banned = await self.data.card.get_card_banned(req.access_code)
-        is_locked = self.data.card.get_card_locked(req.access_code)
+        is_locked = await self.data.card.get_card_locked(req.access_code)
 
         ret = ADBLookupExResponse.from_req(req.head, user_id)
         if is_banned and is_locked:
@@ -189,9 +223,11 @@ class AimedbProtocol(Protocol):
                 self.logger.debug(f"Generated auth token {auth_key}")
                 ret.auth_key = auth_key_full
 
+        if user_id and user_id > 0:
+            await self.data.card.update_card_last_login(req.access_code)
         return ret
 
-    def handle_felica_lookup(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_felica_lookup(self, data: bytes, resp_code: int) -> bytes:
         """
         On official, I think a card has to be registered for this to actually work, but 
         I'm making the executive decision to not implement that and just kick back our
@@ -207,7 +243,7 @@ class AimedbProtocol(Protocol):
         )
         return ADBFelicaLookupResponse.from_req(req.head, ac)
 
-    def handle_felica_register(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_felica_register(self, data: bytes, resp_code: int) -> bytes:
         """
         I've never seen this used.
         """
@@ -215,14 +251,14 @@ class AimedbProtocol(Protocol):
         ac = self.data.card.to_access_code(req.idm)
         
         if self.config.server.allow_user_registration:
-            user_id = self.data.user.create_user()
+            user_id = await self.data.user.create_user()
 
             if user_id is None:
                 self.logger.error("Failed to register user!")
                 user_id = -1
 
             else:
-                card_id = self.data.card.create_card(user_id, ac)
+                card_id = await self.data.card.create_card(user_id, ac)
 
                 if card_id is None:
                     self.logger.error("Failed to register card!")
@@ -237,12 +273,14 @@ class AimedbProtocol(Protocol):
                 f"Registration blocked!: access code {ac} (IDm: {req.idm} PMm: {req.pmm})"
             )
 
+        if user_id > 0:
+            await self.data.card.update_card_last_login(ac)
         return ADBFelicaLookupResponse.from_req(req.head, ac)
 
-    def handle_felica_lookup_ex(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_felica_lookup_ex(self, data: bytes, resp_code: int) -> bytes:
         req = ADBFelicaLookup2Request(data)
         access_code = self.data.card.to_access_code(req.idm)
-        user_id = self.data.card.get_user_id_from_card(access_code=access_code)
+        user_id = await self.data.card.get_user_id_from_card(access_code=access_code)
 
         if user_id is None:
             user_id = -1
@@ -261,9 +299,11 @@ class AimedbProtocol(Protocol):
                 self.logger.debug(f"Generated auth token {auth_key}")
                 resp.auth_key = auth_key_full
         
+        if user_id and user_id > 0:
+            await self.data.card.update_card_last_login(access_code)
         return resp
 
-    def handle_campaign_clear(self, data: bytes, resp_code: int) -> ADBBaseResponse:
+    async def handle_campaign_clear(self, data: bytes, resp_code: int) -> ADBBaseResponse:
         req = ADBCampaignClearRequest(data)
 
         resp = ADBCampaignClearResponse.from_req(req.head)
@@ -271,19 +311,19 @@ class AimedbProtocol(Protocol):
         # We don't support campaign stuff
         return resp
 
-    def handle_register(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_register(self, data: bytes, resp_code: int) -> bytes:
         req = ADBLookupRequest(data)
         user_id = -1
 
         if self.config.server.allow_user_registration:
-            user_id = self.data.user.create_user()
+            user_id = await self.data.user.create_user()
 
             if user_id is None:
                 self.logger.error("Failed to register user!")
                 user_id = -1
 
             else:
-                card_id = self.data.card.create_card(user_id, req.access_code)
+                card_id = await self.data.card.create_card(user_id, req.access_code)
 
                 if card_id is None:
                     self.logger.error("Failed to register card!")
@@ -302,20 +342,23 @@ class AimedbProtocol(Protocol):
         if resp.user_id <= 0:
             resp.head.status = ADBStatus.BAN_SYS # Closest we can get to a "You cannot register"
 
+        else:
+            await self.data.card.update_card_last_login(req.access_code)
+
         return resp
 
     # TODO: Save these in some capacity, as deemed relevant
-    def handle_status_log(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_status_log(self, data: bytes, resp_code: int) -> bytes:
         req = ADBStatusLogRequest(data)
         self.logger.info(f"User {req.aime_id} logged {req.status.name} event")
         return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
 
-    def handle_log(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_log(self, data: bytes, resp_code: int) -> bytes:
         req = ADBLogRequest(data)
         self.logger.info(f"User {req.aime_id} logged {req.status.name} event, credit_ct: {req.credit_ct} bet_ct: {req.bet_ct} won_ct: {req.won_ct}")
         return ADBBaseResponse(resp_code, 0x20, 1, req.head.game_id, req.head.store_id, req.head.keychip_id, req.head.protocol_ver)
 
-    def handle_log_ex(self, data: bytes, resp_code: int) -> bytes:
+    async def handle_log_ex(self, data: bytes, resp_code: int) -> bytes:
         req = ADBLogExRequest(data)
         strs = []
         self.logger.info(f"Recieved {req.num_logs} or {len(req.logs)} logs")
@@ -324,43 +367,3 @@ class AimedbProtocol(Protocol):
             self.logger.debug(f"User {req.logs[x].aime_id} logged {req.logs[x].status.name} event, credit_ct: {req.logs[x].credit_ct} bet_ct: {req.logs[x].bet_ct} won_ct: {req.logs[x].won_ct}")
         return ADBLogExResponse.from_req(req.head)
 
-    def handle_goodbye(self, data: bytes, resp_code: int) -> None:
-        self.logger.info(f"goodbye from {self.transport.getPeer().host}")
-        self.transport.loseConnection()
-        return
-
-class AimedbFactory(Factory):
-    protocol = AimedbProtocol
-
-    def __init__(self, cfg: CoreConfig) -> None:
-        self.config = cfg
-        log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
-        self.logger = logging.getLogger("aimedb")
-
-        fileHandler = TimedRotatingFileHandler(
-            "{0}/{1}.log".format(self.config.server.log_dir, "aimedb"),
-            when="d",
-            backupCount=10,
-        )
-        fileHandler.setFormatter(log_fmt)
-
-        consoleHandler = logging.StreamHandler()
-        consoleHandler.setFormatter(log_fmt)
-
-        self.logger.addHandler(fileHandler)
-        self.logger.addHandler(consoleHandler)
-
-        self.logger.setLevel(self.config.aimedb.loglevel)
-        coloredlogs.install(
-            level=cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str
-        )
-
-        if self.config.aimedb.key == "":
-            self.logger.error("Please set 'key' field in your config file.")
-            exit(1)
-
-        self.logger.info(f"Ready on port {self.config.aimedb.port}")
-
-    def buildProtocol(self, addr):
-        return AimedbProtocol(self.config)

core/allnet.py

@@ -1,20 +1,24 @@
-from typing import Dict, List, Any, Optional, Tuple, Union, Final
-import logging, coloredlogs
-from logging.handlers import TimedRotatingFileHandler
-from twisted.web.http import Request
-from datetime import datetime
 import pytz
 import base64
 import zlib
 import json
+import yaml
+import logging
+import coloredlogs
+import urllib.parse
+import math
+from typing import Dict, List, Any, Optional, Union, Final
+from logging.handlers import TimedRotatingFileHandler
+from starlette.requests import Request
+from starlette.responses import PlainTextResponse
+from starlette.applications import Starlette
+from starlette.routing import Route
+from datetime import datetime
 from enum import Enum
 from Crypto.PublicKey import RSA
 from Crypto.Hash import SHA
 from Crypto.Signature import PKCS1_v1_5
-from time import strptime
+from os import path, environ, mkdir, access, W_OK
-from os import path
-import urllib.parse
-import math
 
 from .config import CoreConfig
 from .utils import Utils
@@ -90,8 +94,8 @@ class DLI_STATUS(Enum):
             return cls.UNKNOWN
 
 class AllnetServlet:
+    allnet_registry: Dict[str, Any] = {}
     def __init__(self, core_cfg: CoreConfig, cfg_folder: str):
-        super().__init__()
         self.config = core_cfg
         self.config_folder = cfg_folder
         self.data = Data(core_cfg)
@@ -120,25 +124,22 @@ class AllnetServlet:
             )
             self.logger.initialized = True
 
-        plugins = Utils.get_all_titles()
+    def startup(self) -> None:
+        self.logger.info(f"Ready on port {self.config.allnet.port if self.config.allnet.standalone else self.config.server.port}")
+        if not TitleServlet.title_registry:
+            TitleServlet(self.config, self.config_folder)
 
-        if len(plugins) == 0:
+    async def handle_poweron(self, request: Request):
-            self.logger.error("No games detected!")
-
-        self.logger.info(
-            f"Serving {len(TitleServlet.title_registry)} game codes port {core_cfg.allnet.port}"
-        )
-
-    def handle_poweron(self, request: Request, _: Dict):
         request_ip = Utils.get_ip_addr(request)
-        pragma_header = request.getHeader('Pragma')
+        pragma_header = request.headers.get('Pragma', "")
         is_dfi = pragma_header is not None and pragma_header == "DFI"
+        data = await request.body()
         
         try:
             if is_dfi:
-                req_urlencode = self.from_dfi(request.content.getvalue())
+                req_urlencode = self.from_dfi(data)
             else:
-                req_urlencode = request.content.getvalue().decode()
+                req_urlencode = data
 
             req_dict = self.allnet_req_to_dict(req_urlencode)
             if req_dict is None:
@@ -155,7 +156,7 @@ class AllnetServlet:
         except AllnetRequestException as e:
             if e.message != "":
                 self.logger.error(e)
-            return b""
+            return PlainTextResponse()
 
         if req.format_ver == 3:
             resp = AllnetPowerOnResponse3(req.token)
@@ -166,42 +167,42 @@ class AllnetServlet:
 
         self.logger.debug(f"Allnet request: {vars(req)}")
 
-        machine = self.data.arcade.get_machine(req.serial)        
+        machine = await self.data.arcade.get_machine(req.serial)        
         if machine is None and not self.config.server.allow_unregistered_serials:
             msg = f"Unrecognised serial {req.serial} attempted allnet auth from {request_ip}."
-            self.data.base.log_event(
+            await self.data.base.log_event(
                 "allnet", "ALLNET_AUTH_UNKNOWN_SERIAL", logging.WARN, msg
             )
             self.logger.warning(msg)
 
             resp.stat = ALLNET_STAT.bad_machine.value
             resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-            return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+            return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
         if machine is not None:
-            arcade = self.data.arcade.get_arcade(machine["arcade"])
+            arcade = await self.data.arcade.get_arcade(machine["arcade"])
             if self.config.server.check_arcade_ip:
                 if arcade["ip"] and arcade["ip"] is not None and arcade["ip"] != req.ip:
                     msg = f"Serial {req.serial} attempted allnet auth from bad IP {req.ip} (expected {arcade['ip']})."
-                    self.data.base.log_event(
+                    await self.data.base.log_event(
                         "allnet", "ALLNET_AUTH_BAD_IP", logging.ERROR, msg
                     )
                     self.logger.warning(msg)
 
                     resp.stat = ALLNET_STAT.bad_shop.value
                     resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-                    return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                    return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
                 
                 elif (not arcade["ip"] or arcade["ip"] is None) and self.config.server.strict_ip_checking:
                     msg = f"Serial {req.serial} attempted allnet auth from bad IP {req.ip}, but arcade {arcade['id']} has no IP set! (strict checking enabled)."
-                    self.data.base.log_event(
+                    await self.data.base.log_event(
                         "allnet", "ALLNET_AUTH_NO_SHOP_IP", logging.ERROR, msg
                     )
                     self.logger.warning(msg)
 
                     resp.stat = ALLNET_STAT.bad_shop.value
                     resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-                    return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                    return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
 
             country = (
@@ -211,7 +212,7 @@ class AllnetServlet:
                 country = AllnetCountryCode.JAPAN.value
 
             resp.country = country
-            resp.place_id = arcade["id"]
+            resp.place_id = f"{arcade['id']:04X}"
             resp.allnet_id = machine["id"]
             resp.name = arcade["name"] if arcade["name"] is not None else ""
             resp.nickname = arcade["nickname"] if arcade["nickname"] is not None else ""
@@ -238,34 +239,40 @@ class AllnetServlet:
         if req.game_id not in TitleServlet.title_registry:
             if not self.config.server.is_develop:
                 msg = f"Unrecognised game {req.game_id} attempted allnet auth from {request_ip}."
-                self.data.base.log_event(
+                await self.data.base.log_event(
                     "allnet", "ALLNET_AUTH_UNKNOWN_GAME", logging.WARN, msg
                 )
                 self.logger.warning(msg)
 
                 resp.stat = ALLNET_STAT.bad_game.value
                 resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
-                return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
+                return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
             else:
                 self.logger.info(
                     f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}"
                 )
-                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
+                resp.uri = f"http://{self.config.server.hostname}:{self.config.server.port}/{req.game_id}/{req.ver.replace('.', '')}/"
-                resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
+                resp.host = f"{self.config.server.hostname}:{self.config.server.port}"
                 
                 resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
                 resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
                 
                 self.logger.debug(f"Allnet response: {resp_str}")
-                return (resp_str + "\n").encode("utf-8")
+                return PlainTextResponse(resp_str + "\n")
 
         
         int_ver = req.ver.replace(".", "")
+        try:
             resp.uri, resp.host = TitleServlet.title_registry[req.game_id].get_allnet_info(req.game_id, int(int_ver), req.serial)
+        except Exception as e:
+            self.logger.error(f"Error running get_allnet_info for {req.game_id} - {e}")
+            resp.stat = ALLNET_STAT.bad_game.value
+            resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+            return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n")
 
         msg = f"{req.serial} authenticated from {request_ip}: {req.game_id} v{req.ver}"
-        self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
+        await self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
         self.logger.info(msg)
 
         resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
@@ -277,18 +284,19 @@ class AllnetServlet:
             request.responseHeaders.addRawHeader('Pragma', 'DFI')
             return self.to_dfi(resp_str)"""
 
-        return resp_str.encode("utf-8")
+        return PlainTextResponse(resp_str)
 
-    def handle_dlorder(self, request: Request, _: Dict):
+    async def handle_dlorder(self, request: Request):
         request_ip = Utils.get_ip_addr(request)
-        pragma_header = request.getHeader('Pragma')
+        pragma_header = request.headers.get('Pragma', "")
         is_dfi = pragma_header is not None and pragma_header == "DFI"
+        data = await request.body()
         
         try:
             if is_dfi:
-                req_urlencode = self.from_dfi(request.content.getvalue())
+                req_urlencode = self.from_dfi(data)
             else:
-                req_urlencode = request.content.getvalue().decode()
+                req_urlencode = data.decode()
 
             req_dict = self.allnet_req_to_dict(req_urlencode)
             if req_dict is None:
@@ -305,7 +313,7 @@ class AllnetServlet:
         except AllnetRequestException as e:
             if e.message != "":
                 self.logger.error(e)
-            return b""
+            return PlainTextResponse()
 
         self.logger.info(
             f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}"
@@ -316,54 +324,54 @@ class AllnetServlet:
             not self.config.allnet.allow_online_updates
             or not self.config.allnet.update_cfg_folder
         ):
-            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
+            return PlainTextResponse(urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n")
 
         else:  # TODO: Keychip check
             if path.exists(
                 f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
             ):
-                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
+                resp.uri = f"http://{self.config.server.hostname}:{self.config.server.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
 
             if path.exists(
                 f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
             ):
-                resp.uri += f"|http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
+                resp.uri += f"|http://{self.config.server.hostname}:{self.config.server.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
 
             self.logger.debug(f"Sending download uri {resp.uri}")
-            self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
+            await self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
 
             res_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
             """if is_dfi:
                 request.responseHeaders.addRawHeader('Pragma', 'DFI')
                 return self.to_dfi(res_str)"""
 
-            return res_str
+            return PlainTextResponse(res_str)
 
-    def handle_dlorder_ini(self, request: Request, match: Dict) -> bytes:
+    async def handle_dlorder_ini(self, request: Request) -> bytes:
-        if "file" not in match:
+        req_file = request.path_params.get("file", "").replace("%0A", "").replace("\n", "")
-            return b""
 
-        req_file = match["file"].replace("%0A", "")
+        if not req_file:
+            return PlainTextResponse(status_code=404)
 
         if path.exists(f"{self.config.allnet.update_cfg_folder}/{req_file}"):
             self.logger.info(f"Request for DL INI file {req_file} from {Utils.get_ip_addr(request)} successful")
-            self.data.base.log_event("allnet", "DLORDER_INI_SENT", logging.INFO, f"{Utils.get_ip_addr(request)} successfully recieved {req_file}")
+            await self.data.base.log_event("allnet", "DLORDER_INI_SENT", logging.INFO, f"{Utils.get_ip_addr(request)} successfully recieved {req_file}")
             
-            return open(
+            return PlainTextResponse(open(
-                f"{self.config.allnet.update_cfg_folder}/{req_file}", "rb"
+                f"{self.config.allnet.update_cfg_folder}/{req_file}", "r", encoding="utf-8"
-            ).read()
+            ).read())
 
         self.logger.info(f"DL INI File {req_file} not found")
-        return b""
+        return PlainTextResponse()
 
-    def handle_dlorder_report(self, request: Request, match: Dict) -> bytes:
+    async def handle_dlorder_report(self, request: Request) -> bytes:
-        req_raw = request.content.getvalue()
+        req_raw = await request.body()
         client_ip = Utils.get_ip_addr(request)
         try:
             req_dict: Dict = json.loads(req_raw)
         except Exception as e:
             self.logger.warning(f"Failed to parse DL Report: {e}")
-            return "NG"
+            return PlainTextResponse("NG")
         
         dl_data_type = DLIMG_TYPE.app
         dl_data = req_dict.get("appimage", {})
@@ -374,24 +382,24 @@ class AllnetServlet:
         
         if dl_data is None or not dl_data:
             self.logger.warning(f"Failed to parse DL Report: Invalid format - contains neither appimage nor optimage")
-            return "NG"
+            return PlainTextResponse("NG")
 
         rep = DLReport(dl_data, dl_data_type)
 
         if not rep.validate():
             self.logger.warning(f"Failed to parse DL Report: Invalid format - {rep.err}")
-            return "NG"
+            return PlainTextResponse("NG")
         
         msg = f"{rep.serial} @ {client_ip} reported {rep.rep_type.name} download state {rep.rf_state.name} for {rep.gd} v{rep.dav}:"\
               f" {rep.tdsc}/{rep.tsc} segments downloaded for working files {rep.wfl} with {rep.dfl if rep.dfl else 'none'} complete."
         
-        self.data.base.log_event("allnet", "DL_REPORT", logging.INFO, msg, dl_data)
+        await self.data.base.log_event("allnet", "DL_REPORT", logging.INFO, msg, dl_data)
         self.logger.info(msg)
 
-        return "OK"
+        return PlainTextResponse("OK")
 
-    def handle_loaderstaterecorder(self, request: Request, match: Dict) -> bytes:
+    async def handle_loaderstaterecorder(self, request: Request) -> bytes:
-        req_data = request.content.getvalue()
+        req_data = await request.body()
         sections = req_data.decode("utf-8").split("\r\n")
         
         req_dict = dict(urllib.parse.parse_qsl(sections[0]))
@@ -403,130 +411,17 @@ class AllnetServlet:
         ip = Utils.get_ip_addr(request)
 
         if serial is None or num_files_dld is None or num_files_to_dl is None or dl_state is None:
-            return "NG".encode()
+            return PlainTextResponse("NG")
 
         self.logger.info(f"LoaderStateRecorder Request from {ip} {serial}: {num_files_dld}/{num_files_to_dl} Files download (State: {dl_state})")
-        return "OK".encode()
+        return PlainTextResponse("OK")
     
-    def handle_alive(self, request: Request, match: Dict) -> bytes:
+    async def handle_alive(self, request: Request) -> bytes:
-        return "OK".encode()
+        return PlainTextResponse("OK")
 
-    def handle_billing_request(self, request: Request, _: Dict):
+    async def handle_naomitest(self, request: Request) -> bytes:
-        req_raw = request.content.getvalue()
-        
-        if request.getHeader('Content-Type') == "application/octet-stream":
-            req_unzip = zlib.decompressobj(-zlib.MAX_WBITS).decompress(req_raw)
-        else:
-            req_unzip = req_raw
-        
-        req_dict = self.billing_req_to_dict(req_unzip)
-        request_ip = Utils.get_ip_addr(request)
-        
-        if req_dict is None:
-            self.logger.error(f"Failed to parse request {request.content.getvalue()}")
-            return b""
-
-        self.logger.debug(f"request {req_dict}")
-
-        rsa = RSA.import_key(open(self.config.billing.signing_key, "rb").read())
-        signer = PKCS1_v1_5.new(rsa)
-        digest = SHA.new()
-        traces: List[TraceData] = []
-        try:
-            req = BillingInfo(req_dict[0])
-        except KeyError as e:
-            self.logger.error(f"Billing request failed to parse: {e}")
-            return f"result=5&linelimit=&message=field is missing or formatting is incorrect\r\n".encode()
-
-        for x in range(1, len(req_dict)):
-            if not req_dict[x]:
-                continue
-            
-            try:
-                tmp = TraceData(req_dict[x])
-                if tmp.trace_type == TraceDataType.CHARGE:
-                    tmp = TraceDataCharge(req_dict[x])
-                elif tmp.trace_type == TraceDataType.EVENT:
-                    tmp = TraceDataEvent(req_dict[x])
-                elif tmp.trace_type == TraceDataType.CREDIT:
-                    tmp = TraceDataCredit(req_dict[x])
-                
-                traces.append(tmp)
-            
-            except KeyError as e:
-                self.logger.warn(f"Tracelog failed to parse: {e}")
-
-        kc_serial_bytes = req.keychipid.encode()
-        
-
-        machine = self.data.arcade.get_machine(req.keychipid)
-        if machine is None and not self.config.server.allow_unregistered_serials:
-            msg = f"Unrecognised serial {req.keychipid} attempted billing checkin from {request_ip} for {req.gameid} v{req.gamever}."
-            self.data.base.log_event(
-                "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg
-            )
-            self.logger.warning(msg)
-
-            return f"result=1&requestno={req.requestno}&message=Keychip Serial bad\r\n".encode()
-
-        msg = (
-            f"Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount "
-            f"{req.playcnt} billing_type {req.billingtype.name} nearfull {req.nearfull} playlimit {req.playlimit}"
-        )
-        self.logger.info(msg)
-        self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, msg)
-        if req.traceleft > 0:
-            self.logger.warn(f"{req.traceleft} unsent tracelogs")
-        kc_playlimit = req.playlimit
-        kc_nearfull = req.nearfull
-
-        while req.playcnt > req.playlimit:
-            kc_playlimit += 1024
-            kc_nearfull += 1024
-
-        playlimit = kc_playlimit
-        nearfull = kc_nearfull + (req.billingtype.value * 0x00010000)
-
-        digest.update(playlimit.to_bytes(4, "little") + kc_serial_bytes)
-        playlimit_sig = signer.sign(digest).hex()
-
-        digest = SHA.new()
-        digest.update(nearfull.to_bytes(4, "little") + kc_serial_bytes)
-        nearfull_sig = signer.sign(digest).hex()
-
-        # TODO: playhistory
-
-        #resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig)
-        resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig, req.requestno, req.protocolver)
-
-        resp_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\r\n"
-
-        self.logger.debug(f"response {vars(resp)}")
-        if req.traceleft > 0:
-            self.logger.info(f"Requesting 20 more of {req.traceleft} unsent tracelogs")
-            return f"result=6&waittime=0&linelimit=20\r\n".encode()
-
-        return resp_str.encode("utf-8")
-
-    def handle_naomitest(self, request: Request, _: Dict) -> bytes:
         self.logger.info(f"Ping from {Utils.get_ip_addr(request)}")
-        return b"naomi ok"
+        return PlainTextResponse("naomi ok")
-
-    def billing_req_to_dict(self, data: bytes):
-        """
-        Parses an billing request string into a python dictionary
-        """
-        try:
-            sections = data.decode("ascii").split("\r\n")
-
-            ret = []
-            for x in sections:
-                ret.append(dict(urllib.parse.parse_qsl(x)))
-            return ret
-
-        except Exception as e:
-            self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
-            return None
 
     def allnet_req_to_dict(self, data: str) -> Optional[List[Dict[str, Any]]]:
         """
@@ -554,6 +449,150 @@ class AllnetServlet:
         zipped = zlib.compress(unzipped)
         return base64.b64encode(zipped)
 
+class BillingServlet:
+    def __init__(self, core_cfg: CoreConfig, cfg_folder: str) -> None:
+        self.config = core_cfg
+        self.config_folder = cfg_folder
+        self.data = Data(core_cfg)
+
+        self.logger = logging.getLogger("billing")
+        if not hasattr(self.logger, "initialized"):
+            log_fmt_str = "[%(asctime)s] Billing | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "billing"),
+                when="d",
+                backupCount=10,
+            )
+            fileHandler.setFormatter(log_fmt)
+
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
+
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(core_cfg.allnet.loglevel)
+            coloredlogs.install(
+                level=core_cfg.billing.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.initialized = True
+    
+    def startup(self) -> None:
+        self.logger.info(f"Ready on port {self.config.billing.port if self.config.billing.standalone else self.config.server.port}")
+
+    def billing_req_to_dict(self, data: bytes):
+        """
+        Parses an billing request string into a python dictionary
+        """
+        try:
+            sections = data.decode("ascii").split("\r\n")
+
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
+
+        except Exception as e:
+            self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
+            return None
+    
+    async def handle_billing_request(self, request: Request):
+        req_raw = await request.body()
+        
+        if request.headers.get('Content-Type', '') == "application/octet-stream":
+            req_unzip = zlib.decompressobj(-zlib.MAX_WBITS).decompress(req_raw)
+        else:
+            req_unzip = req_raw
+        
+        req_dict = self.billing_req_to_dict(req_unzip)
+        request_ip = Utils.get_ip_addr(request)
+        
+        if req_dict is None:
+            self.logger.error(f"Failed to parse request {req_raw}")
+            return PlainTextResponse()
+
+        self.logger.debug(f"request {req_dict}")
+
+        rsa = RSA.import_key(open(self.config.billing.signing_key, "rb").read())
+        signer = PKCS1_v1_5.new(rsa)
+        digest = SHA.new()
+        traces: List[TraceData] = []
+        try:
+            req = BillingInfo(req_dict[0])
+        except KeyError as e:
+            self.logger.error(f"Billing request failed to parse: {e}")
+            return PlainTextResponse("result=5&linelimit=&message=field is missing or formatting is incorrect\r\n")
+
+        for x in range(1, len(req_dict)):
+            if not req_dict[x]:
+                continue
+            
+            try:
+                tmp = TraceData(req_dict[x])
+                if tmp.trace_type == TraceDataType.CHARGE:
+                    tmp = TraceDataCharge(req_dict[x])
+                elif tmp.trace_type == TraceDataType.EVENT:
+                    tmp = TraceDataEvent(req_dict[x])
+                elif tmp.trace_type == TraceDataType.CREDIT:
+                    tmp = TraceDataCredit(req_dict[x])
+                
+                traces.append(tmp)
+            
+            except KeyError as e:
+                self.logger.warn(f"Tracelog failed to parse: {e}")
+
+        kc_serial_bytes = req.keychipid.encode()
+        
+
+        machine = await self.data.arcade.get_machine(req.keychipid)
+        if machine is None and not self.config.server.allow_unregistered_serials:
+            msg = f"Unrecognised serial {req.keychipid} attempted billing checkin from {request_ip} for {req.gameid} v{req.gamever}."
+            await self.data.base.log_event(
+                "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg
+            )
+            self.logger.warning(msg)
+
+            return PlainTextResponse(f"result=1&requestno={req.requestno}&message=Keychip Serial bad\r\n")
+
+        msg = (
+            f"Billing checkin from {request_ip}: game {req.gameid} ver {req.gamever} keychip {req.keychipid} playcount "
+            f"{req.playcnt} billing_type {req.billingtype.name} nearfull {req.nearfull} playlimit {req.playlimit}"
+        )
+        self.logger.info(msg)
+        await self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, msg)
+        if req.traceleft > 0:
+            self.logger.warn(f"{req.traceleft} unsent tracelogs")
+        kc_playlimit = req.playlimit
+        kc_nearfull = req.nearfull
+
+        while req.playcnt > req.playlimit:
+            kc_playlimit += 1024
+            kc_nearfull += 1024
+
+        playlimit = kc_playlimit
+        nearfull = kc_nearfull + (req.billingtype.value * 0x00010000)
+
+        digest.update(playlimit.to_bytes(4, "little") + kc_serial_bytes)
+        playlimit_sig = signer.sign(digest).hex()
+
+        digest = SHA.new()
+        digest.update(nearfull.to_bytes(4, "little") + kc_serial_bytes)
+        nearfull_sig = signer.sign(digest).hex()
+
+        # TODO: playhistory
+
+        resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig, req.requestno, req.protocolver)
+
+        resp_str = urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\r\n"
+
+        self.logger.debug(f"response {vars(resp)}")
+        if req.traceleft > 0:
+            self.logger.info(f"Requesting 20 more of {req.traceleft} unsent tracelogs")
+            return PlainTextResponse("result=6&waittime=0&linelimit=20\r\n")
+        
+        return PlainTextResponse(resp_str)
 
 class AllnetPowerOnRequest:
     def __init__(self, req: Dict) -> None:
@@ -613,7 +652,6 @@ class AllnetPowerOnResponse3(AllnetPowerOnResponse):
         self.minute = None
         self.second = None
 
-
 class AllnetPowerOnResponse2(AllnetPowerOnResponse):
     def __init__(self) -> None:
         super().__init__()
@@ -623,7 +661,6 @@ class AllnetPowerOnResponse2(AllnetPowerOnResponse):
         self.timezone = "+09:00"
         self.res_class = "PowerOnResponseV2"
 
-
 class AllnetDownloadOrderRequest:
     def __init__(self, req: Dict) -> None:
         self.game_id = req.get("game_id", "")
@@ -631,7 +668,6 @@ class AllnetDownloadOrderRequest:
         self.serial = req.get("serial", "")
         self.encode = req.get("encode", "")
 
-
 class AllnetDownloadOrderResponse:
     def __init__(self, stat: int = 1, serial: str = "", uri: str = "") -> None:
         self.stat = stat
@@ -781,7 +817,6 @@ class BillingResponse:
         # playhistory -> YYYYMM/C:...
         # YYYY -> 4 digit year, MM -> 2 digit month, C -> Playcount during that period
 
-
 class AllnetRequestException(Exception):
     def __init__(self, message="") -> None:
         self.message = message
@@ -849,3 +884,48 @@ class DLReport:
             return False
         
         return True
+
+cfg_dir = environ.get("DIANA_CFG_DIR", "config")
+cfg: CoreConfig = CoreConfig()
+if path.exists(f"{cfg_dir}/core.yaml"):
+    cfg.update(yaml.safe_load(open(f"{cfg_dir}/core.yaml")))
+
+if not path.exists(cfg.server.log_dir):
+    mkdir(cfg.server.log_dir)
+
+if not access(cfg.server.log_dir, W_OK):
+    print(
+        f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+    )
+    exit(1)
+
+billing = BillingServlet(cfg, cfg_dir)
+app_billing = Starlette(
+    cfg.server.is_develop, 
+    [        
+        Route("/request", billing.handle_billing_request, methods=["POST"]),
+        Route("/request/", billing.handle_billing_request, methods=["POST"]),
+    ],
+    on_startup=[billing.startup]
+)
+
+allnet = AllnetServlet(cfg, cfg_dir)
+route_lst = [
+    Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]),
+    Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]),
+    Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]),
+    Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]),
+    Route("/naomitest.html", allnet.handle_naomitest),
+]
+
+if cfg.allnet.allow_online_updates:
+    route_lst += [
+        Route("/report-api/Report", allnet.handle_dlorder_report, methods=["POST"]),
+        Route("/dl/ini/{file:str}", allnet.handle_dlorder_ini),
+    ]
+
+app_allnet = Starlette(
+    cfg.server.is_develop, 
+    route_lst,
+    on_startup=[allnet.startup]
+)

core/app.py

@@ -0,0 +1,92 @@
+import yaml
+import logging
+import coloredlogs
+from logging.handlers import TimedRotatingFileHandler
+from starlette.routing import Route
+from starlette.requests import Request
+from starlette.applications import Starlette
+from starlette.responses import PlainTextResponse
+from os import environ, path, mkdir, W_OK, access
+from typing import List
+
+from core import CoreConfig, TitleServlet, MuchaServlet, AllnetServlet, BillingServlet, AimedbServlette
+from core.frontend import FrontendServlet
+
+async def dummy_rt(request: Request):
+    return PlainTextResponse("Service OK")
+
+cfg_dir = environ.get("ARTEMIS_CFG_DIR", "config")
+cfg: CoreConfig = CoreConfig()
+if path.exists(f"{cfg_dir}/core.yaml"):
+    cfg.update(yaml.safe_load(open(f"{cfg_dir}/core.yaml")))
+
+if not path.exists(cfg.server.log_dir):
+    mkdir(cfg.server.log_dir)
+
+if not access(cfg.server.log_dir, W_OK):
+    print(
+        f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+    )
+    exit(1)
+
+logger = logging.getLogger("core")
+log_fmt_str = "[%(asctime)s] Core | %(levelname)s | %(message)s"
+log_fmt = logging.Formatter(log_fmt_str)
+
+fileHandler = TimedRotatingFileHandler(
+    "{0}/{1}.log".format(cfg.server.log_dir, "core"), when="d", backupCount=10
+)
+fileHandler.setFormatter(log_fmt)
+
+consoleHandler = logging.StreamHandler()
+consoleHandler.setFormatter(log_fmt)
+
+logger.addHandler(fileHandler)
+logger.addHandler(consoleHandler)
+
+log_lv = logging.DEBUG if cfg.server.is_develop else logging.INFO
+logger.setLevel(log_lv)
+coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
+
+logger.info(f"Artemis starting in {'develop' if cfg.server.is_develop else 'production'} mode")
+
+title = TitleServlet(cfg, cfg_dir) # This has to be loaded first to load plugins
+mucha = MuchaServlet(cfg, cfg_dir)
+
+route_lst: List[Route] = [
+    # Mucha
+    Route("/mucha_front/boardauth.do", mucha.handle_boardauth, methods=["POST"]),
+    Route("/mucha_front/updatacheck.do", mucha.handle_updatecheck, methods=["POST"]),
+    Route("/mucha_front/downloadstate.do", mucha.handle_dlstate, methods=["POST"]),
+    # General
+    Route("/", dummy_rt),
+    Route("/robots.txt", FrontendServlet.robots)
+]
+
+if not cfg.billing.standalone:
+    billing = BillingServlet(cfg, cfg_dir)
+    route_lst += [
+        Route("/request", billing.handle_billing_request, methods=["POST"]),
+        Route("/request/", billing.handle_billing_request, methods=["POST"]),
+    ]
+
+if not cfg.allnet.standalone:
+    allnet = AllnetServlet(cfg, cfg_dir)
+    route_lst += [
+        Route("/sys/servlet/PowerOn", allnet.handle_poweron, methods=["GET", "POST"]),
+        Route("/sys/servlet/DownloadOrder", allnet.handle_dlorder, methods=["GET", "POST"]),
+        Route("/sys/servlet/LoaderStateRecorder", allnet.handle_loaderstaterecorder, methods=["GET", "POST"]),
+        Route("/sys/servlet/Alive", allnet.handle_alive, methods=["GET", "POST"]),
+        Route("/naomitest.html", allnet.handle_naomitest),
+    ]
+    
+    if cfg.allnet.allow_online_updates:
+        route_lst += [
+            Route("/report-api/Report", allnet.handle_dlorder_report, methods=["POST"]),
+            Route("/dl/ini/{file:str}", allnet.handle_dlorder_ini),
+        ]
+
+for code, game in title.title_registry.items():
+    route_lst += game.get_routes()
+
+app = Starlette(cfg.server.is_develop, route_lst)

core/config.py

@@ -1,17 +1,49 @@
 import logging, os
 from typing import Any
 
-
 class ServerConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
     def listen_address(self) -> str:
+        """
+        Address Artemis will bind to and listen on
+        """
         return CoreConfig.get_config_field(
             self.__config, "core", "server", "listen_address", default="127.0.0.1"
         )
     
+    @property
+    def hostname(self) -> str:
+        """
+        Hostname sent to games
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "hostname", default="localhost"
+        )
+    
+    @property
+    def port(self) -> int:
+        """
+        Port the game will listen on
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "port", default=80
+        )
+
+    @property
+    def ssl_key(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "ssl_key", default="cert/title.key"
+        )
+
+    @property
+    def ssl_cert(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "ssl_cert", default="cert/title.pem"
+        )
+
     @property
     def allow_user_registration(self) -> bool:
         return CoreConfig.get_config_field(
@@ -43,9 +75,23 @@ class ServerConfig:
         )
 
     @property
-    def threading(self) -> bool:
+    def proxy_port(self) -> int:
+        """
+        What port the proxy is listening on. This will be sent instead of 'port' if 
+        is_using_proxy is True and this value is non-zero
+        """
         return CoreConfig.get_config_field(
-            self.__config, "core", "server", "threading", default=False
+            self.__config, "core", "server", "proxy_port", default=0
+        )
+
+    @property
+    def proxy_port_ssl(self) -> int:
+        """
+        What port the proxy is listening for secure connections on. This will be sent 
+        instead of 'port' if is_using_proxy is True and this value is non-zero
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "proxy_port_ssl", default=0
         )
 
     @property
@@ -66,7 +112,6 @@ class ServerConfig:
             self.__config, "core", "server", "strict_ip_checking", default=False
         )
 
-
 class TitleConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
@@ -79,36 +124,6 @@ class TitleConfig:
             )
         )
 
-    @property
-    def hostname(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "hostname", default="localhost"
-        )
-
-    @property
-    def port(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "port", default=8080
-        )
-        
-    @property
-    def port_ssl(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "port_ssl", default=0
-        )
-
-    @property
-    def ssl_key(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "ssl_key", default="cert/title.key"
-        )
-
-    @property
-    def ssl_cert(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "title", "ssl_cert", default="cert/title.pem"
-        )
-
     @property
     def reboot_start_time(self) -> str:
         return CoreConfig.get_config_field(
@@ -121,7 +136,6 @@ class TitleConfig:
             self.__config, "core", "title", "reboot_end_time", default=""
         )
 
-
 class DatabaseConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
@@ -159,7 +173,7 @@ class DatabaseConfig:
     @property
     def protocol(self) -> str:
         return CoreConfig.get_config_field(
-            self.__config, "core", "database", "type", default="mysql"
+            self.__config, "core", "database", "protocol", default="mysql"
         )
 
     @property
@@ -176,16 +190,6 @@ class DatabaseConfig:
             )
         )
 
-    @property
-    def user_table_autoincrement_start(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config,
-            "core",
-            "database",
-            "user_table_autoincrement_start",
-            default=10000,
-        )
-
     @property
     def enable_memcached(self) -> bool:
         return CoreConfig.get_config_field(
@@ -198,13 +202,12 @@ class DatabaseConfig:
             self.__config, "core", "database", "memcached_host", default="localhost"
         )
 
-
 class FrontendConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
-    def enable(self) -> int:
+    def enable(self) -> bool:
         return CoreConfig.get_config_field(
             self.__config, "core", "frontend", "enable", default=False
         )
@@ -212,7 +215,7 @@ class FrontendConfig:
     @property
     def port(self) -> int:
         return CoreConfig.get_config_field(
-            self.__config, "core", "frontend", "port", default=8090
+            self.__config, "core", "frontend", "port", default=8080
         )
 
     @property
@@ -223,17 +226,20 @@ class FrontendConfig:
             )
         )
     
+    @property
+    def secret(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "frontend", "secret", default=""
+        )
 
 class AllnetConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
-    def loglevel(self) -> int:
+    def standalone(self) -> bool:
-        return CoreConfig.str_to_loglevel(
+        return CoreConfig.get_config_field(
-            CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "standalone", default=False
-                self.__config, "core", "allnet", "loglevel", default="info"
-            )
         )
     
     @property
@@ -243,9 +249,11 @@ class AllnetConfig:
         )
 
     @property
-    def ip_check(self) -> bool:
+    def loglevel(self) -> int:
-        return CoreConfig.get_config_field(
+        return CoreConfig.str_to_loglevel(
-            self.__config, "core", "allnet", "ip_check", default=False
+            CoreConfig.get_config_field(
+                self.__config, "core", "allnet", "loglevel", default="info"
+            )
         )
 
     @property
@@ -260,11 +268,24 @@ class AllnetConfig:
             self.__config, "core", "allnet", "update_cfg_folder", default=""
         )
 
-
 class BillingConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
     
+    @property
+    def standalone(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "standalone", default=True
+        )
+
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "billing", "loglevel", default="info"
+            )
+        )
+
     @property
     def port(self) -> int:
         return CoreConfig.get_config_field(
@@ -289,11 +310,22 @@ class BillingConfig:
             self.__config, "core", "billing", "signing_key", default="cert/billing.key"
         )
 
-
 class AimedbConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
+    @property
+    def enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "enable", default=True
+        )
+
+    @property
+    def listen_address(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "listen_address", default=""
+        )
+
     @property
     def loglevel(self) -> int:
         return CoreConfig.str_to_loglevel(
@@ -326,17 +358,10 @@ class AimedbConfig:
             self.__config, "core", "aimedb", "id_lifetime_seconds", default=86400
         )
 
-
 class MuchaConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
-    @property
-    def enable(self) -> int:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "mucha", "enable", default=False
-        )
-
     @property
     def loglevel(self) -> int:
         return CoreConfig.str_to_loglevel(
@@ -345,13 +370,6 @@ class MuchaConfig:
             )
         )
 
-    @property
-    def hostname(self) -> str:
-        return CoreConfig.get_config_field(
-            self.__config, "core", "mucha", "hostname", default="localhost"
-        )
-
-
 class CoreConfig(dict):
     def __init__(self) -> None:
         self.server = ServerConfig(self)
@@ -374,6 +392,19 @@ class CoreConfig(dict):
         else:
             return logging.INFO
     
+    @classmethod
+    def loglevel_to_str(cls, level: int) -> str:
+        if level == logging.ERROR:
+            return "error"
+        elif level == logging.WARN:
+            return "warn"
+        elif level == logging.INFO:
+            return "info"
+        elif level == logging.DEBUG:
+            return "debug"
+        else:
+            return "notset"
+
     @classmethod
     def get_config_field(
         cls, __config: dict, module, *path: str, default: Any = ""

core/data/alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

core/data/alembic/alembic.ini

@@ -0,0 +1,64 @@
+# A generic, single database configuration.
+
+[alembic]
+script_location=.
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to migrations//versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat migrations//versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

core/data/alembic/env.py

@@ -0,0 +1,81 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+from core.data.schema.base import metadata
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    raise Exception('Not implemented or configured!')
+
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    ini_section = config.get_section(config.config_ini_section)
+    overrides = context.get_x_argument(as_dictionary=True)
+    for override in overrides:
+        ini_section[override] = overrides[override]
+
+    connectable = engine_from_config(
+        ini_section,
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            compare_type=True,
+            compare_server_default=True,
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

core/data/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

core/data/alembic/versions/835b862f9bf0_initial_migration.py

@@ -0,0 +1,24 @@
+"""Initial Migration
+
+Revision ID: 835b862f9bf0
+Revises: 
+Create Date: 2024-01-09 13:06:10.787432
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '835b862f9bf0'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

core/data/alembic/versions/d8950c7ce2fc_remove_old_db_mgmt_system.py

@@ -0,0 +1,29 @@
+"""Remove old db mgmt system
+
+Revision ID: d8950c7ce2fc
+Revises: 835b862f9bf0
+Create Date: 2024-01-09 13:43:51.381175
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd8950c7ce2fc'
+down_revision = '835b862f9bf0'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_table("schema_versions")
+
+
+def downgrade():
+    op.create_table(
+        "schema_versions",
+        sa.Column("game", sa.String(4), primary_key=True, nullable=False),
+        sa.Column("version", sa.Integer, nullable=False, server_default="1"),
+        mysql_charset="utf8mb4",
+    )

core/data/database.py

@@ -1,13 +1,14 @@
 import logging, coloredlogs
-from typing import Optional, Dict, List
+from typing import Optional
 from sqlalchemy.orm import scoped_session, sessionmaker
-from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
 from logging.handlers import TimedRotatingFileHandler
-import importlib, os
+import os
 import secrets, string
 import bcrypt
 from hashlib import sha256
+import alembic.config
+import glob
 
 from core.config import CoreConfig
 from core.data.schema import *
@@ -15,7 +16,6 @@ from core.utils import Utils
 
 
 class Data:
-    current_schema_version = 6
     engine = None
     session = None
     user = None
@@ -77,281 +77,173 @@ class Data:
             )
             self.logger.handler_set = True  # type: ignore
 
+    def __alembic_cmd(self, command: str, *args: str) -> None:
+        old_dir = os.path.abspath(os.path.curdir)
+        base_dir = os.path.join(os.path.abspath(os.path.curdir), 'core', 'data', 'alembic')
+        alembicArgs = [
+            "-c",
+            os.path.join(base_dir, "alembic.ini"),
+            "-x",
+            f"script_location={base_dir}",
+            "-x",
+            f"sqlalchemy.url={self.__url}",
+            command,
+        ]
+        alembicArgs.extend(args)
+        os.chdir(base_dir)
+        alembic.config.main(argv=alembicArgs)
+        os.chdir(old_dir)
+
     def create_database(self):
         self.logger.info("Creating databases...")
-        try:
+        metadata.create_all(
-            metadata.create_all(self.__engine.connect())
+            self.engine,
-        except SQLAlchemyError as e:
+            checkfirst=True,
-            self.logger.error(f"Failed to create databases! {e}")
-            return
-
-        games = Utils.get_all_titles()
-        for game_dir, game_mod in games.items():
-            try:
-                if hasattr(game_mod, "database") and hasattr(
-                    game_mod, "current_schema_version"
-                ):
-                    game_mod.database(self.config)
-                metadata.create_all(self.__engine.connect())
-
-                self.base.touch_schema_ver(
-                    game_mod.current_schema_version, game_mod.game_codes[0]
         )
 
-            except Exception as e:
+        for _, mod in Utils.get_all_titles().items():
-                self.logger.warning(
-                    f"Could not load database schema from {game_dir} - {e}"
-                )
-
-        self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
-        self.base.set_schema_ver(self.current_schema_version)
-
-        self.logger.info(
-            f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
-        )
-        self.user.reset_autoincrement(
-            self.config.database.user_table_autoincrement_start
-        )
-
-    def recreate_database(self):
-        self.logger.info("Dropping all databases...")
-        self.base.execute("SET FOREIGN_KEY_CHECKS=0")
-        try:
-            metadata.drop_all(self.__engine.connect())
-        except SQLAlchemyError as e:
-            self.logger.error(f"Failed to drop databases! {e}")
-            return
-
-        for root, dirs, files in os.walk("./titles"):
-            for dir in dirs:
-                if not dir.startswith("__"):
-                    try:
-                        mod = importlib.import_module(f"titles.{dir}")
-
-                        try:
             if hasattr(mod, "database"):
                 mod.database(self.config)
-                            metadata.drop_all(self.__engine.connect())
+                metadata.create_all(
-
+                    self.engine,
-                        except Exception as e:
+                    checkfirst=True,
-                            self.logger.warning(
-                                f"Could not load database schema from {dir} - {e}"
                 )
 
-                    except ImportError as e:
+        # Stamp the end revision as if alembic had created it, so it can take off after this.
-                        self.logger.warning(
+        self.__alembic_cmd(
-                            f"Failed to load database schema dir {dir} - {e}"
+            "stamp",
-                        )
+            "head",
-            break
-
-        self.base.execute("SET FOREIGN_KEY_CHECKS=1")
-
-        self.create_database()
-
-    def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
-        old_ver = self.base.get_schema_ver(game)
-        sql = ""
-        if version is None:
-            if not game == "CORE":
-                titles = Utils.get_all_titles()
-
-                for folder, mod in titles.items():
-                    if not mod.game_codes[0] == game:
-                        continue
-
-                    if hasattr(mod, "current_schema_version"):
-                        version = mod.current_schema_version
-
-                    else:
-                        self.logger.warning(
-                            f"current_schema_version not found for {folder}"
         )
 
-            else:
+    def schema_upgrade(self, ver: str = None):
-                version = self.current_schema_version
+        self.__alembic_cmd(
-
+            "upgrade",
-        if version is None:
+            "head" if not ver else ver,
-            self.logger.warning(
-                f"Could not determine latest version for {game}, please specify --version"
         )
 
-        if old_ver is None:
+    def schema_downgrade(self, ver: str):
-            self.logger.error(
+        self.__alembic_cmd(
-                f"Schema for game {game} does not exist, did you run the creation script?"
+            "downgrade",
+            ver,
         )
-            return
 
-        if old_ver == version:
+    async def create_owner(self, email: Optional[str] = None, code: Optional[str] = "00000000000000000000") -> None:
-            self.logger.info(
-                f"Schema for game {game} is already version {old_ver}, nothing to do"
-            )
-            return
-
-        if action == "upgrade":
-            for x in range(old_ver, version):
-                if not os.path.exists(
-                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql"
-                ):
-                    self.logger.error(
-                        f"Could not find {action} script {game.upper()}_{x + 1}_{action}.sql in core/data/schema/versions folder"
-                    )
-                    return
-
-                with open(
-                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql",
-                    "r",
-                    encoding="utf-8",
-                ) as f:
-                    sql = f.read()
-
-                result = self.base.execute(sql)
-                if result is None:
-                    self.logger.error("Error execuing sql script!")
-                    return None
-
-        else:
-            for x in range(old_ver, version, -1):
-                if not os.path.exists(
-                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql"
-                ):
-                    self.logger.error(
-                        f"Could not find {action} script {game.upper()}_{x - 1}_{action}.sql in core/data/schema/versions folder"
-                    )
-                    return
-
-                with open(
-                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql",
-                    "r",
-                    encoding="utf-8",
-                ) as f:
-                    sql = f.read()
-
-                result = self.base.execute(sql)
-                if result is None:
-                    self.logger.error("Error execuing sql script!")
-                    return None
-
-        result = self.base.set_schema_ver(version, game)
-        if result is None:
-            self.logger.error("Error setting version in schema_version table!")
-            return None
-
-        self.logger.info(f"Successfully migrated {game} to schema version {version}")
-
-    def create_owner(self, email: Optional[str] = None) -> None:
         pw = "".join(
             secrets.choice(string.ascii_letters + string.digits) for i in range(20)
         )
         hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt())
 
-        user_id = self.user.create_user(email=email, permission=255, password=hash)
+        user_id = await self.user.create_user(username="sysowner", email=email, password=hash.decode(), permission=255)
         if user_id is None:
             self.logger.error(f"Failed to create owner with email {email}")
             return
 
-        card_id = self.card.create_card(user_id, "00000000000000000000")
+        card_id = await self.card.create_card(user_id, code)
         if card_id is None:
             self.logger.error(f"Failed to create card for owner with id {user_id}")
             return
 
         self.logger.warning(
-            f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!"
+            f"Successfully created owner with email {email}, access code {code}, and password {pw} Make sure to change this password and assign a real card ASAP!"
         )
     
-    def migrate_card(self, old_ac: str, new_ac: str, should_force: bool) -> None:
+    async def migrate(self) -> None:
-        if old_ac == new_ac:
+        exist = await self.base.execute("SELECT * FROM alembic_version")
-            self.logger.error("Both access codes are the same!")
+        if exist is not None:
+            self.logger.warn("No need to migrate as you have already migrated to alembic. If you are trying to upgrade the schema, use `upgrade` instead!")
             return
         
-        new_card = self.card.get_card_by_access_code(new_ac)
+        self.logger.info("Upgrading to latest with legacy system")
-        if new_card is None:
+        if not await self.legacy_upgrade():
-            self.card.update_access_code(old_ac, new_ac)
+            self.logger.warn("No need to migrate as you have already deleted the old schema_versions system. If you are trying to upgrade the schema, use `upgrade` instead!")
             return
+        self.logger.info("Done")
         
-        if not should_force:
+        self.logger.info("Stamp with initial revision")
-            self.logger.warning(
+        self.__alembic_cmd(
-                f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
+            "stamp",
-                f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
+            "835b862f9bf0",
         )
-            return
 
-        self.logger.info(
+        self.logger.info("Upgrade")
-            f"All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
+        self.__alembic_cmd(
+            "upgrade",
+            "head",
         )
-        self.card.delete_card(new_card["id"])
-        self.card.update_access_code(old_ac, new_ac)
     
-        hanging_user = self.user.get_user(new_card["user"])
+    async def legacy_upgrade(self) -> bool:
-        if hanging_user["password"] is None:
+        vers = await self.base.execute("SELECT * FROM schema_versions")
-            self.logger.info(f"Delete hanging user {hanging_user['id']}")
+        if vers is None:
-            self.user.delete_user(hanging_user["id"])
+            self.logger.warn("Cannot legacy upgrade, schema_versions table unavailable!")
+            return False
         
-    def delete_hanging_users(self) -> None:
+        db_vers = {}
-        """
+        vers_list = vers.fetchall()
-        Finds and deletes users that have not registered for the webui that have no cards assocated with them.
+        for x in vers_list:
-        """
+            db_vers[x['game']] = x['version']
-        unreg_users = self.user.get_unregistered_users()
-        if unreg_users is None:
-            self.logger.error("Error occoured finding unregistered users")
         
-        for user in unreg_users:
+        core_now_ver = int(db_vers['CORE']) + 1
-            cards = self.card.get_user_cards(user["id"])
+        while os.path.exists(f"core/data/schema/versions/CORE_{core_now_ver}_upgrade.sql"):
-            if cards is None:
+            with open(f"core/data/schema/versions/CORE_{core_now_ver}_upgrade.sql", "r") as f:
-                self.logger.error(f"Error getting cards for user {user['id']}")
+                result = await self.base.execute(f.read())
-                continue
                 
-            if not cards:
-                self.logger.info(f"Delete hanging user {user['id']}")
-                self.user.delete_user(user["id"])
-
-    def autoupgrade(self) -> None:
-        all_game_versions = self.base.get_all_schema_vers()
-        if all_game_versions is None:
-            self.logger.warning("Failed to get schema versions")
-            return
-
-        all_games = Utils.get_all_titles()
-        all_games_list: Dict[str, int] = {}
-        for _, mod in all_games.items():
-            if hasattr(mod, "current_schema_version"):
-                all_games_list[mod.game_codes[0]] = mod.current_schema_version
-
-        for x in all_game_versions:
-            failed = False
-            game = x["game"].upper()
-            update_ver = int(x["version"])
-            latest_ver = all_games_list.get(game, 1)
-            if game == "CORE":
-                latest_ver = self.current_schema_version
-
-            if update_ver == latest_ver:
-                self.logger.info(f"{game} is already latest version")
-                continue
-
-            for y in range(update_ver + 1, latest_ver + 1):
-                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
-                    with open(
-                        f"core/data/schema/versions/{game}_{y}_upgrade.sql",
-                        "r",
-                        encoding="utf-8",
-                    ) as f:
-                        sql = f.read()
-
-                    result = self.base.execute(sql)
                 if result is None:
-                        self.logger.error(
+                    self.logger.error(f"Invalid upgrade script CORE_{core_now_ver}_upgrade.sql")
-                            f"Error execuing sql script for game {game} v{y}!"
-                        )
-                        failed = True
                     break
-                else:
-                    self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
-                    failed = True
 
-            if not failed:
+                result = await self.base.execute(f"UPDATE schema_versions SET version = {core_now_ver} WHERE game = 'CORE'")
-                self.base.set_schema_ver(latest_ver, game)
+                if result is None:
+                    self.logger.error(f"Failed to update schema version for CORE to {core_now_ver}")
+                    break
             
-    def show_versions(self) -> None:
+            self.logger.info(f"Upgrade CORE to version {core_now_ver}")
-        all_game_versions = self.base.get_all_schema_vers()
+            core_now_ver += 1
-        for ver in all_game_versions:
+        
-            self.logger.info(f"{ver['game']} -> v{ver['version']}")
+        for _, mod in Utils.get_all_titles().items():
+            game_codes = getattr(mod, "game_codes", [])
+            for game in game_codes:
+                if game not in db_vers:
+                    self.logger.warn(f"{game} does not have an antry in schema_versions, skipping")
+                    continue
+
+                now_ver = int(db_vers[game]) + 1
+                while os.path.exists(f"core/data/schema/versions/{game}_{now_ver}_upgrade.sql"):
+                    with open(f"core/data/schema/versions/{game}_{now_ver}_upgrade.sql", "r") as f:
+                        result = await self.base.execute(f.read())
+                        
+                        if result is None:
+                            self.logger.error(f"Invalid upgrade script {game}_{now_ver}_upgrade.sql")
+                            break
+
+                        result = await self.base.execute(f"UPDATE schema_versions SET version = {now_ver} WHERE game = '{game}'")
+                        if result is None:
+                            self.logger.error(f"Failed to update schema version for {game} to {now_ver}")
+                            break
+
+                    self.logger.info(f"Upgrade {game} to version {now_ver}")
+                    now_ver += 1
+        
+        return True
+
+    async def create_revision(self, message: str) -> None:
+        if not message:
+            self.logger.info("Message is required for create-revision")
+            return
+        
+        self.__alembic_cmd(
+            "revision",
+            "-m",
+            message,
+        )
+
+    async def create_revision_auto(self, message: str) -> None:
+        if not message:
+            self.logger.info("Message is required for create-revision")
+            return
+        
+        self.__alembic_cmd(
+            "revision",
+            "--autogenerate",
+            "-m",
+            message,
+        )

core/data/schema/arcade.py

@@ -69,7 +69,7 @@ arcade_owner = Table(
 
 
 class ArcadeData(BaseData):
-    def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]:
+    async def get_machine(self, serial: str = None, id: int = None) -> Optional[Row]:
         if serial is not None:
             serial = serial.replace("-", "")
             if len(serial) == 11:
@@ -89,12 +89,12 @@ class ArcadeData(BaseData):
             self.logger.error(f"{__name__ }: Need either serial or ID to look up!")
             return None
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def put_machine(
+    async def put_machine(
         self,
         arcade_id: int,
         serial: str = "",
@@ -110,13 +110,13 @@ class ArcadeData(BaseData):
             arcade=arcade_id, keychip=serial, board=board, game=game, is_cab=is_cab
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def set_machine_serial(self, machine_id: int, serial: str) -> None:
+    async def set_machine_serial(self, machine_id: int, serial: str) -> None:
-        result = self.execute(
+        result = await self.execute(
             machine.update(machine.c.id == machine_id).values(keychip=serial)
         )
         if result is None:
@@ -125,8 +125,8 @@ class ArcadeData(BaseData):
             )
         return result.lastrowid
 
-    def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
+    async def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
-        result = self.execute(
+        result = await self.execute(
             machine.update(machine.c.id == machine_id).values(board=boardid)
         )
         if result is None:
@@ -134,21 +134,21 @@ class ArcadeData(BaseData):
                 f"Failed to update board id for machine {machine_id} -> {boardid}"
             )
 
-    def get_arcade(self, id: int) -> Optional[Row]:
+    async def get_arcade(self, id: int) -> Optional[Row]:
         sql = arcade.select(arcade.c.id == id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
     
-    def get_arcade_machines(self, id: int) -> Optional[List[Row]]:
+    async def get_arcade_machines(self, id: int) -> Optional[List[Row]]:
         sql = machine.select(machine.c.arcade == id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def put_arcade(
+    async def put_arcade(
         self,
         name: str,
         nickname: str = None,
@@ -171,42 +171,42 @@ class ArcadeData(BaseData):
             regional_id=regional_id,
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_arcades_managed_by_user(self, user_id: int) -> Optional[List[Row]]:
+    async def get_arcades_managed_by_user(self, user_id: int) -> Optional[List[Row]]:
         sql = select(arcade).join(arcade_owner, arcade_owner.c.arcade == arcade.c.id).where(arcade_owner.c.user == user_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchall()
     
-    def get_manager_permissions(self, user_id: int, arcade_id: int) -> Optional[int]:
+    async def get_manager_permissions(self, user_id: int, arcade_id: int) -> Optional[int]:
         sql = select(arcade_owner.c.permissions).where(and_(arcade_owner.c.user == user_id, arcade_owner.c.arcade == arcade_id))
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchone()
 
-    def get_arcade_owners(self, arcade_id: int) -> Optional[Row]:
+    async def get_arcade_owners(self, arcade_id: int) -> Optional[Row]:
         sql = select(arcade_owner).where(arcade_owner.c.arcade == arcade_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
+    async def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
         sql = insert(arcade_owner).values(arcade=arcade_id, user=user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
-    def format_serial(
+    async def format_serial(
         self, platform_code: str, platform_rev: int, serial_num: int, append: int = 4152
     ) -> str:
         return f"{platform_code}{platform_rev:02d}A{serial_num:04d}{append:04d}"  # 0x41 = A, 0x52 = R
@@ -217,16 +217,16 @@ class ArcadeData(BaseData):
         
         return True
 
-    def get_arcade_by_name(self, name: str) -> Optional[List[Row]]:
+    async def get_arcade_by_name(self, name: str) -> Optional[List[Row]]:
         sql = arcade.select(or_(arcade.c.name.like(f"%{name}%"), arcade.c.nickname.like(f"%{name}%")))
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def get_arcades_by_ip(self, ip: str) -> Optional[List[Row]]:
+    async def get_arcades_by_ip(self, ip: str) -> Optional[List[Row]]:
         sql = arcade.select().where(arcade.c.ip == ip)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()

core/data/schema/base.py

@@ -15,14 +15,6 @@ from core.config import CoreConfig
 
 metadata = MetaData()
 
-schema_ver = Table(
-    "schema_versions",
-    metadata,
-    Column("game", String(4), primary_key=True, nullable=False),
-    Column("version", Integer, nullable=False, server_default="1"),
-    mysql_charset="utf8mb4",
-)
-
 event_log = Table(
     "event_log",
     metadata,
@@ -43,11 +35,11 @@ class BaseData:
         self.conn = conn
         self.logger = logging.getLogger("database")
 
-    def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
+    async def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
         res = None
 
         try:
-            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}")
+            self.logger.debug(f"SQL Execute: {''.join(str(sql).splitlines())}")
             res = self.conn.execute(text(sql), opts)
 
         except SQLAlchemyError as e:
@@ -82,52 +74,7 @@ class BaseData:
         """
         return randrange(10000, 9999999)
 
-    def get_all_schema_vers(self) -> Optional[List[Row]]:
+    async def log_event(
-        sql = select(schema_ver)
-
-        result = self.execute(sql)
-        if result is None:
-            return None
-        return result.fetchall()
-
-    def get_schema_ver(self, game: str) -> Optional[int]:
-        sql = select(schema_ver).where(schema_ver.c.game == game)
-
-        result = self.execute(sql)
-        if result is None:
-            return None
-
-        row = result.fetchone()
-        if row is None:
-            return None
-
-        return row["version"]
-    
-    def touch_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version=schema_ver.c.version)
-
-        result = self.execute(conflict)
-        if result is None:
-            self.logger.error(
-                f"Failed to update schema version for game {game} (v{ver})"
-            )
-            return None
-        return result.lastrowid
-
-    def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version=ver)
-
-        result = self.execute(conflict)
-        if result is None:
-            self.logger.error(
-                f"Failed to update schema version for game {game} (v{ver})"
-            )
-            return None
-        return result.lastrowid
-
-    def log_event(
         self, system: str, type: str, severity: int, message: str, details: Dict = {}
     ) -> Optional[int]:
         sql = event_log.insert().values(
@@ -137,7 +84,7 @@ class BaseData:
             message=message,
             details=json.dumps(details),
         )
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             self.logger.error(
@@ -147,9 +94,9 @@ class BaseData:
 
         return result.lastrowid
 
-    def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
+    async def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
         sql = event_log.select().limit(entries).all()
-        result = self.execute(sql)
+        result = await self.execute(sql)
 
         if result is None:
             return None

core/data/schema/card.py

@@ -27,91 +27,101 @@ aime_card = Table(
 
 
 class CardData(BaseData):
-    def get_card_by_access_code(self, access_code: str) -> Optional[Row]:
+    async def get_card_by_access_code(self, access_code: str) -> Optional[Row]:
         sql = aime_card.select(aime_card.c.access_code == access_code)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def get_card_by_id(self, card_id: int) -> Optional[Row]:
+    async def get_card_by_id(self, card_id: int) -> Optional[Row]:
         sql = aime_card.select(aime_card.c.id == card_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchone()
 
-    def update_access_code(self, old_ac: str, new_ac: str) -> None:
+    async def update_access_code(self, old_ac: str, new_ac: str) -> None:
         sql = aime_card.update(aime_card.c.access_code == old_ac).values(
             access_code=new_ac
         )
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(
                 f"Failed to change card access code from {old_ac} to {new_ac}"
             )
 
-    def get_user_id_from_card(self, access_code: str) -> Optional[int]:
+    async def get_user_id_from_card(self, access_code: str) -> Optional[int]:
         """
         Given a 20 digit access code as a string, get the user id associated with that card
         """
-        card = self.get_card_by_access_code(access_code)
+        card = await self.get_card_by_access_code(access_code)
         if card is None:
             return None
 
         return int(card["user"])
 
-    def get_card_banned(self, access_code: str) -> Optional[bool]:
+    async def get_card_banned(self, access_code: str) -> Optional[bool]:
         """
         Given a 20 digit access code as a string, check if the card is banned
         """
-        card = self.get_card_by_access_code(access_code)
+        card = await self.get_card_by_access_code(access_code)
         if card is None:
             return None
         if card["is_banned"]:
             return True
         return False
-    def get_card_locked(self, access_code: str) -> Optional[bool]:
+    
+    async def get_card_locked(self, access_code: str) -> Optional[bool]:
         """
         Given a 20 digit access code as a string, check if the card is locked
         """
-        card = self.get_card_by_access_code(access_code)
+        card = await self.get_card_by_access_code(access_code)
         if card is None:
             return None
         if card["is_locked"]:
             return True
         return False
 
-    def delete_card(self, card_id: int) -> None:
+    async def delete_card(self, card_id: int) -> None:
         sql = aime_card.delete(aime_card.c.id == card_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(f"Failed to delete card with id {card_id}")
 
-    def get_user_cards(self, aime_id: int) -> Optional[List[Row]]:
+    async def get_user_cards(self, aime_id: int) -> Optional[List[Row]]:
         """
         Returns all cards owned by a user
         """
         sql = aime_card.select(aime_card.c.user == aime_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def create_card(self, user_id: int, access_code: str) -> Optional[int]:
+    async def create_card(self, user_id: int, access_code: str) -> Optional[int]:
         """
         Given a aime_user id and a 20 digit access code as a string, create a card and return the ID if successful
         """
         sql = aime_card.insert().values(user=user_id, access_code=access_code)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.lastrowid
 
+    async def update_card_last_login(self, access_code: str) -> None:
+        sql = aime_card.update(aime_card.c.access_code == access_code).values(
+            last_login_date=func.now()
+        )
+        
+        result = await self.execute(sql)
+        if result is None:
+            self.logger.warn(f"Failed to update last login time for {access_code}")
+    
     def to_access_code(self, luid: str) -> str:
         """
         Given a felica cards internal 16 hex character luid, convert it to a 0-padded 20 digit access code as a string

core/data/schema/user.py

@@ -1,4 +1,3 @@
-from enum import Enum
 from typing import Optional, List
 from sqlalchemy import Table, Column
 from sqlalchemy.types import Integer, String, TIMESTAMP
@@ -24,15 +23,8 @@ aime_user = Table(
     mysql_charset="utf8mb4",
 )
 
-
-class PermissionBits(Enum):
-    PermUser = 1
-    PermMod = 2
-    PermSysAdmin = 4
-
-
 class UserData(BaseData):
-    def create_user(
+    async def create_user(
         self,
         id: int = None,
         username: str = None,
@@ -60,20 +52,20 @@ class UserData(BaseData):
             username=username, email=email, password=password, permissions=permission
         )
 
-        result = self.execute(conflict)
+        result = await self.execute(conflict)
         if result is None:
             return None
         return result.lastrowid
 
-    def get_user(self, user_id: int) -> Optional[Row]:
+    async def get_user(self, user_id: int) -> Optional[Row]:
         sql = select(aime_user).where(aime_user.c.id == user_id)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchone()
 
-    def check_password(self, user_id: int, passwd: bytes = None) -> bool:
+    async def check_password(self, user_id: int, passwd: bytes = None) -> bool:
-        usr = self.get_user(user_id)
+        usr = await self.get_user(user_id)
         if usr is None:
             return False
 
@@ -85,39 +77,46 @@ class UserData(BaseData):
 
         return bcrypt.checkpw(passwd, usr["password"].encode())
 
-    def reset_autoincrement(self, ai_value: int) -> None:
+    async def delete_user(self, user_id: int) -> None:
-        # ALTER TABLE isn't in sqlalchemy so we do this the ugly way
-        sql = f"ALTER TABLE aime_user AUTO_INCREMENT={ai_value}"
-        self.execute(sql)
-
-    def delete_user(self, user_id: int) -> None:
         sql = aime_user.delete(aime_user.c.id == user_id)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             self.logger.error(f"Failed to delete user with id {user_id}")
 
-    def get_unregistered_users(self) -> List[Row]:
+    async def get_unregistered_users(self) -> List[Row]:
         """
         Returns a list of users who have not registered with the webui. They may or may not have cards.
         """
         sql = select(aime_user).where(aime_user.c.password == None)
 
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return None
         return result.fetchall()
 
-    def find_user_by_email(self, email: str) -> Row:
+    async def find_user_by_email(self, email: str) -> Row:
         sql = select(aime_user).where(aime_user.c.email == email)
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchone()
 
-    def find_user_by_username(self, username: str) -> List[Row]:
+    async def find_user_by_username(self, username: str) -> List[Row]:
         sql = aime_user.select(aime_user.c.username.like(f"%{username}%"))
-        result = self.execute(sql)
+        result = await self.execute(sql)
         if result is None:
             return False
         return result.fetchall()
+
+    async def change_password(self, user_id: int, new_passwd: str) -> bool:
+        sql = aime_user.update(aime_user.c.id == user_id).values(password = new_passwd)
+
+        result = await self.execute(sql)
+        return result is not None
+
+    async def change_username(self, user_id: int, new_name: str) -> bool:
+        sql = aime_user.update(aime_user.c.id == user_id).values(username = new_name)
+
+        result = await self.execute(sql)
+        return result is not None

core/data/schema/versions/SDDT_5_rollback.sql

@@ -1,8 +1,8 @@
 SET FOREIGN_KEY_CHECKS=0;
 
 ALTER TABLE ongeki_user_event_point DROP COLUMN version;
-ALTER TABLE ongeki_user_event_point DROP COLUMN rank;
+ALTER TABLE ongeki_user_event_point DROP COLUMN `rank`;
-ALTER TABLE ongeki_user_event_point DROP COLUMN type;
+ALTER TABLE ongeki_user_event_point DROP COLUMN `type`;
 ALTER TABLE ongeki_user_event_point DROP COLUMN date;
 
 ALTER TABLE ongeki_user_tech_event DROP COLUMN version;

core/data/schema/versions/SDDT_6_upgrade.sql

@@ -1,8 +1,8 @@
 SET FOREIGN_KEY_CHECKS=0;
 
 ALTER TABLE ongeki_user_event_point ADD COLUMN version INTEGER NOT NULL;
-ALTER TABLE ongeki_user_event_point ADD COLUMN rank INTEGER;
+ALTER TABLE ongeki_user_event_point ADD COLUMN `rank` INTEGER;
-ALTER TABLE ongeki_user_event_point ADD COLUMN type INTEGER NOT NULL;
+ALTER TABLE ongeki_user_event_point ADD COLUMN `type` INTEGER NOT NULL;
 ALTER TABLE ongeki_user_event_point ADD COLUMN date VARCHAR(25);
 
 ALTER TABLE ongeki_user_tech_event ADD COLUMN version INTEGER NOT NULL;
@@ -17,7 +17,7 @@ CREATE TABLE ongeki_tech_event_ranking (
     version INT NOT NULL,
     date VARCHAR(25),
     eventId INT NOT NULL,
-	rank INT,
+    `rank` INT,
     totalPlatinumScore INT NOT NULL,
     totalTechScore INT NOT NULL,
     UNIQUE KEY ongeki_tech_event_ranking_uk (user, eventId),
@@ -88,11 +88,11 @@ CREATE TABLE ongeki_static_client_testmode (
 
 CREATE TABLE ongeki_static_game_point (
     id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
-	type INT NOT NULL,
+    `type` INT NOT NULL,
     cost INT NOT NULL,
     startDate VARCHAR(25) NOT NULL DEFAULT "2000-01-01 05:00:00.0",
     endDate VARCHAR(25) NOT NULL DEFAULT "2099-01-01 05:00:00.0",
-	UNIQUE KEY ongeki_static_game_point_uk (type)
+    UNIQUE KEY ongeki_static_game_point_uk (`type`)
 );
 
 SET FOREIGN_KEY_CHECKS=1;

core/frontend.py

@@ -1,28 +1,23 @@
 import logging, coloredlogs
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Union, Optional
-from twisted.web import resource
+from starlette.requests import Request
-from twisted.web.util import redirectTo
+from starlette.routing import Route, Mount
-from twisted.web.http import Request
+from starlette.responses import Response, PlainTextResponse, RedirectResponse
+from starlette.applications import Starlette
 from logging.handlers import TimedRotatingFileHandler
-from twisted.web.server import Session
-from zope.interface import Interface, Attribute, implementer
-from twisted.python.components import registerAdapter
 import jinja2
 import bcrypt
 import re
+import jwt
+import yaml
+from base64 import b64decode
 from enum import Enum
-from urllib import parse
+from datetime import datetime, timezone
+from os import path, environ, mkdir, W_OK, access
 
 from core import CoreConfig, Utils
 from core.data import Data
 
-
-class IUserSession(Interface):
-    userId = Attribute("User's ID")
-    current_ip = Attribute("User's current ip address")
-    permissions = Attribute("User's permission level")
-    ongeki_version = Attribute("User's selected Ongeki Version")
-
 class PermissionOffset(Enum):
     USER = 0 # Regular user
     USERMOD = 1 # Can moderate other users
@@ -31,31 +26,38 @@ class PermissionOffset(Enum):
     # 4 - 6 reserved for future use
     OWNER = 7 # Can do anything
 
-@implementer(IUserSession)
+class ShopPermissionOffset(Enum):
-class UserSession(object):
+    VIEW = 0 # View info and cabs
-    def __init__(self, session):
+    BOOKKEEP = 1 # View bookeeping info
-        self.userId = 0
+    EDITOR = 2 # Can edit name, settings
-        self.current_ip = "0.0.0.0"
+    REGISTRAR = 3 # Can add cabs
-        self.permissions = 0
+    # 4 - 6 reserved for future use
-        self.ongeki_version = 7
+    OWNER = 7 # Can do anything
 
+class ShopOwner():
+    def __init__(self, usr_id: int = 0, usr_name: str = "", perms: int = 0) -> None:
+        self.user_id = usr_id
+        self.username = usr_name
+        self.permissions = perms
 
-class FrontendServlet(resource.Resource):
+class UserSession():
-    def getChild(self, name: bytes, request: Request):
+    def __init__(self, usr_id: int = 0, ip: str = "", perms: int = 0, ongeki_ver: int = 7):
-        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {name.decode()}")
+        self.user_id = usr_id
-        if name == b"":
+        self.current_ip = ip
-            return self
+        self.permissions = perms
-        return resource.Resource.getChild(self, name, request)
+        self.ongeki_version = ongeki_ver
 
+class FrontendServlet():
     def __init__(self, cfg: CoreConfig, config_dir: str) -> None:
         self.config = cfg
         log_fmt_str = "[%(asctime)s] Frontend | %(levelname)s | %(message)s"
         log_fmt = logging.Formatter(log_fmt_str)
-        self.logger = logging.getLogger("frontend")
         self.environment = jinja2.Environment(loader=jinja2.FileSystemLoader("."))
-        self.game_list: List[Dict[str, str]] = []
+        self.game_list: Dict[str, Dict[str, Any]] = {}
-        self.children: Dict[str, Any] = {}
+        self.sn_cvt: Dict[str, str] = {}
         
+        self.logger = logging.getLogger("frontend")
+        if not hasattr(self.logger, "inited"):
             fileHandler = TimedRotatingFileHandler(
                 "{0}/{1}.log".format(self.config.server.log_dir, "frontend"),
                 when="d",
@@ -73,206 +75,351 @@ class FrontendServlet(resource.Resource):
             coloredlogs.install(
                 level=cfg.frontend.loglevel, logger=self.logger, fmt=log_fmt_str
             )
-        registerAdapter(UserSession, Session, IUserSession)
 
-        fe_game = FE_Game(cfg, self.environment)
+            self.logger.inited = True
+        
         games = Utils.get_all_titles()
         for game_dir, game_mod in games.items():
-            if hasattr(game_mod, "frontend"):
+            if hasattr(game_mod, "frontend") and hasattr(game_mod, "index") and hasattr(game_mod, "game_codes"):
                 try:
+                    if game_mod.index.is_game_enabled(game_mod.game_codes[0], self.config, config_dir):
                         game_fe = game_mod.frontend(cfg, self.environment, config_dir)
-                    self.game_list.append({"url": game_dir, "name": game_fe.nav_name})
+                        self.game_list[game_fe.nav_name] = {"url": f"/{game_dir}", "class": game_fe }
-                    fe_game.putChild(game_dir.encode(), game_fe)
+                        
+                        if hasattr(game_fe, "SN_PREFIX") and hasattr(game_fe, "NETID_PREFIX"):
+                            if len(game_fe.SN_PREFIX) == len(game_fe.NETID_PREFIX):
+                                for x in range(len(game_fe.SN_PREFIX)):
+                                    self.sn_cvt[game_fe.SN_PREFIX[x]] = game_fe.NETID_PREFIX[x]
 
                 except Exception as e:
                     self.logger.error(
                         f"Failed to import frontend from {game_dir} because {e}"
                     )
 
+
         self.environment.globals["game_list"] = self.game_list
-        self.putChild(b"gate", FE_Gate(cfg, self.environment))
+        self.environment.globals["sn_cvt"] = self.sn_cvt
-        self.putChild(b"user", FE_User(cfg, self.environment))
+        self.base = FE_Base(cfg, self.environment)
-        self.putChild(b"sys", FE_System(cfg, self.environment))
+        self.gate = FE_Gate(cfg, self.environment)
-        self.putChild(b"arcade", FE_Arcade(cfg, self.environment))
+        self.user = FE_User(cfg, self.environment)
-        self.putChild(b"cab", FE_Machine(cfg, self.environment))
+        self.system = FE_System(cfg, self.environment)
-        self.putChild(b"game", fe_game)
+        self.arcade = FE_Arcade(cfg, self.environment)
+        self.machine = FE_Machine(cfg, self.environment)
     
-        self.logger.info(
+    def get_routes(self) -> List[Route]:        
-            f"Ready on port {self.config.frontend.port} serving {len(fe_game.children)} games"
+        g_routes = []
-        )
+        for nav_name, g_data in self.environment.globals["game_list"].items():
+            g_routes.append(Mount(g_data['url'], routes=g_data['class'].get_routes()))
+        return [
+            Route("/", self.base.render_GET, methods=['GET']),
+            Mount("/user", routes=[
+                Route("/", self.user.render_GET, methods=['GET']),
+                Route("/{user_id:int}", self.user.render_GET, methods=['GET']),
+                Route("/update.pw", self.user.render_POST, methods=['POST']),
+                Route("/update.name", self.user.update_username, methods=['POST']),
+                Route("/edit.card", self.user.edit_card, methods=['POST']),
+                Route("/add.card", self.user.add_card, methods=['POST']),
+                Route("/logout", self.user.render_logout, methods=['GET']),
+            ]),
+            Mount("/gate", routes=[
+                Route("/", self.gate.render_GET, methods=['GET', 'POST']),
+                Route("/gate.login", self.gate.render_login, methods=['POST']),
+                Route("/gate.create", self.gate.render_create, methods=['POST']),
+                Route("/create", self.gate.render_create_get, methods=['GET']),
+            ]),
+            Mount("/sys", routes=[
+                Route("/", self.system.render_GET, methods=['GET']),
+                Route("/lookup.user", self.system.lookup_user, methods=['GET']),
+                Route("/lookup.shop", self.system.lookup_shop, methods=['GET']),
+            ]),
+            Mount("/shop", routes=[
+                Route("/", self.arcade.render_GET, methods=['GET']),
+                Route("/{shop_id:int}", self.arcade.render_GET, methods=['GET']),
+            ]),
+            Mount("/cab", routes=[
+                Route("/", self.machine.render_GET, methods=['GET']),
+                Route("/{machine_id:int}", self.machine.render_GET, methods=['GET']),
+            ]),
+            Mount("/game", routes=g_routes),            
+            Route("/robots.txt", self.robots)
+        ]
     
-    def render_GET(self, request):
+    def startup(self) -> None:
-        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.uri.decode()}")
+        self.config.update({
-        template = self.environment.get_template("core/frontend/index.jinja")
+            "frontend": {
-        return template.render(
+                "standalone": True, 
-            server_name=self.config.server.name,
+                "loglevel": CoreConfig.loglevel_to_str(self.config.frontend.loglevel), 
-            title=self.config.server.name,
+                "secret": self.config.frontend.secret
-            game_list=self.game_list,
+            }
-            sesh=vars(IUserSession(request.getSession())),
+        })
-        ).encode("utf-16")
+        self.logger.info(f"Serving {len(self.game_list)} games")
     
+    @classmethod    
+    async def robots(cls, request: Request) -> PlainTextResponse:
+        return PlainTextResponse("User-agent: *\nDisallow: /\n\nUser-agent: AdsBot-Google\nDisallow: /")
 
-class FE_Base(resource.Resource):
+class FE_Base():
     """
     A Generic skeleton class that all frontend handlers should inherit from
     Initializes the environment, data, logger, config, and sets isLeaf to true
     It is expected that game implementations of this class overwrite many of these
     """
-
-    isLeaf = True
-
     def __init__(self, cfg: CoreConfig, environment: jinja2.Environment) -> None:
         self.core_config = cfg
         self.data = Data(cfg)
         self.logger = logging.getLogger("frontend")
         self.environment = environment
-        self.nav_name = "nav_name"
+        self.nav_name = "index"
     
+    async def render_GET(self, request: Request):
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.url}")
+        template = self.environment.get_template("core/templates/index.jinja")
+        sesh = self.validate_session(request)
+        resp = Response(template.render(
+            server_name=self.core_config.server.name,
+            title=self.core_config.server.name,
+            game_list=self.environment.globals["game_list"],
+            sesh=vars(sesh) if sesh is not None else vars(UserSession()),
+        ), media_type="text/html; charset=utf-8")
+        
+        if sesh is None:
+            resp.delete_cookie("DIANA_SESH")
+        return resp
+    
+    def get_routes(self) -> List[Route]:
+        return []
+    
+    @classmethod
+    def test_perm(cls, permission: int, offset: Union[PermissionOffset, ShopPermissionOffset]) -> bool:
+        logging.getLogger('frontend').debug(f"{permission} vs {1 << offset.value}")
+        return permission & 1 << offset.value == 1 << offset.value
+    
+    @classmethod
+    def test_perm_minimum(cls, permission: int, offset: Union[PermissionOffset, ShopPermissionOffset]) -> bool:
+        return permission >= 1 << offset.value
+    
+    def decode_session(self, token: str) -> UserSession:
+        sesh = UserSession()
+        if not token: return sesh
+        try:
+            tk = jwt.decode(token, b64decode(self.core_config.frontend.secret), options={"verify_signature": True}, algorithms=["HS256"])
+            sesh.user_id = tk['user_id']
+            sesh.current_ip = tk['current_ip']
+            sesh.permissions = tk['permissions']
+            
+            if sesh.user_id <= 0:
+                self.logger.error("User session failed to validate due to an invalid ID!")
+                return UserSession()
+            return sesh
+        except jwt.ExpiredSignatureError:
+            self.logger.error("User session failed to validate due to an expired signature!")
+            return sesh
+        except jwt.InvalidSignatureError:
+            self.logger.error("User session failed to validate due to an invalid signature!")
+            return sesh
+        except jwt.DecodeError as e:
+            self.logger.error(f"User session failed to decode! {e}")
+            return sesh
+        except jwt.InvalidTokenError as e:
+            self.logger.error(f"User session is invalid! {e}")
+            return sesh
+        except KeyError as e:
+            self.logger.error(f"{e} missing from User session!")
+            return UserSession()
+        except Exception as e:
+            self.logger.error(f"Unknown exception occoured when decoding User session! {e}")
+            return UserSession()
+    
+    def validate_session(self, request: Request) -> Optional[UserSession]:
+        sesh = request.cookies.get('DIANA_SESH', "")
+        if not sesh:
+            return None
+        
+        usr_sesh = self.decode_session(sesh)
+        req_ip = Utils.get_ip_addr(request)
+        
+        if usr_sesh.current_ip != req_ip:
+            self.logger.error(f"User session failed to validate due to mismatched IPs! {usr_sesh.current_ip} -> {req_ip}")
+            return None
+
+        if usr_sesh.permissions <= 0 or usr_sesh.permissions > 255:
+            self.logger.error(f"User session failed to validate due to an invalid permission value! {usr_sesh.permissions}")
+            return None
+        
+        return usr_sesh
+    
+    def encode_session(self, sesh: UserSession, exp_seconds: int = 86400) -> str:
+        try:
+            return jwt.encode({ "user_id": sesh.user_id, "current_ip": sesh.current_ip, "permissions": sesh.permissions, "ongeki_version": sesh.ongeki_version, "exp": int(datetime.now(tz=timezone.utc).timestamp()) + exp_seconds }, b64decode(self.core_config.frontend.secret), algorithm="HS256")
+        except jwt.InvalidKeyError:
+            self.logger.error("Failed to encode User session because the secret is invalid!")
+            return ""
+        except Exception as e:
+            self.logger.error(f"Unknown exception occoured when encoding User session! {e}")
+            return ""
 
 class FE_Gate(FE_Base):
-    def render_GET(self, request: Request):
+    async def render_GET(self, request: Request):
-        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.uri.decode()}")
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.url.path}")
-        uri: str = request.uri.decode()
 
-        sesh = request.getSession()
+        usr_sesh = self.validate_session(request)
-        usr_sesh = IUserSession(sesh)
+        if usr_sesh and usr_sesh.user_id > 0:
-        if usr_sesh.userId > 0:
+            return RedirectResponse("/user/", 303)
-            return redirectTo(b"/user", request)
         
-        if uri.startswith("/gate/create"):
-            return self.create_user(request)
         
-        if b"e" in request.args:
+        if "e" in request.query_params:
             try:
-                err = int(request.args[b"e"][0].decode())
+                err = int(request.query_params.get("e", ["0"])[0])
             except Exception:
                 err = 0
 
         else:
             err = 0
 
-        template = self.environment.get_template("core/frontend/gate/gate.jinja")
+        template = self.environment.get_template("core/templates/gate/gate.jinja")
-        return template.render(
+        resp = Response(template.render(
             title=f"{self.core_config.server.name} | Login Gate",
             error=err,
-            sesh=vars(usr_sesh),
+            sesh=vars(UserSession()),
-        ).encode("utf-16")
+        ), media_type="text/html; charset=utf-8")
+        resp.delete_cookie("DIANA_SESH")
+        return resp
     
-    def render_POST(self, request: Request):
+    async def render_login(self, request: Request):
-        uri = request.uri.decode()
         ip = Utils.get_ip_addr(request)
+        frm = await request.form()
+        access_code: str = frm.get("access_code", None)
+        if not access_code:
+            return RedirectResponse("/gate/?e=1", 303)
         
-        if uri == "/gate/gate.login":
+        passwd: bytes = frm.get("passwd", "").encode()
-            access_code: str = request.args[b"access_code"][0].decode()
-            passwd: bytes = request.args[b"passwd"][0]
         if passwd == b"":
             passwd = None
 
-            uid = self.data.card.get_user_id_from_card(access_code)
+        uid = await self.data.card.get_user_id_from_card(access_code)
-            user = self.data.user.get_user(uid)
         if uid is None:
-                return redirectTo(b"/gate?e=1", request)
+            self.logger.debug(f"Failed to find user for card {access_code}")
+            return RedirectResponse("/gate/?e=1", 303)
+
+        user = await self.data.user.get_user(uid)
+        if user is None:
+            self.logger.error(f"Failed to load user {uid}")
+            return RedirectResponse("/gate/?e=1", 303)
 
         if passwd is None:
-                sesh = self.data.user.check_password(uid)
+            sesh = await self.data.user.check_password(uid)
 
             if sesh is not None:
-                    return redirectTo(
+                return RedirectResponse(f"/gate/create?ac={access_code}", 303)
-                        f"/gate/create?ac={access_code}".encode(), request
-                    )
-                return redirectTo(b"/gate?e=1", request)
             
-            if not self.data.user.check_password(uid, passwd):
+            return RedirectResponse("/gate/?e=1", 303)
-                return redirectTo(b"/gate?e=1", request)
+
+        if not await self.data.user.check_password(uid, passwd):
+            self.logger.debug(f"Failed password for access code {access_code}")
+            return RedirectResponse("/gate/?e=1", 303)
 
         self.logger.info(f"Successful login of user {uid} at {ip}")
 
-            sesh = request.getSession()
+        sesh = UserSession()
-            usr_sesh = IUserSession(sesh)
+        sesh.user_id = uid
-            usr_sesh.userId = uid
+        sesh.current_ip = ip
-            usr_sesh.current_ip = ip
+        sesh.permissions = user['permissions']
-            usr_sesh.permissions = user['permissions']
         
-            return redirectTo(b"/user", request)
+        usr_sesh = self.encode_session(sesh)
+        self.logger.debug(f"Created session with JWT {usr_sesh}")
+        resp = RedirectResponse("/user/", 303)
+        resp.set_cookie("DIANA_SESH", usr_sesh)
 
-        elif uri == "/gate/gate.create":
+        return resp
-            access_code: str = request.args[b"access_code"][0].decode()
-            username: str = request.args[b"username"][0]
-            email: str = request.args[b"email"][0].decode()
-            passwd: bytes = request.args[b"passwd"][0]
 
-            uid = self.data.card.get_user_id_from_card(access_code)
+    async def render_create(self, request: Request):
+        ip = Utils.get_ip_addr(request)
+        frm = await request.form()
+        access_code: str = frm.get("access_code", "")
+        username: str = frm.get("username", "")
+        email: str = frm.get("email", "")
+        passwd: bytes = frm.get("passwd", "").encode()
+
+        if not access_code or not username or not email or not passwd:
+            return RedirectResponse("/gate/?e=1", 303)
+
+        uid = await self.data.card.get_user_id_from_card(access_code)
         if uid is None:
-                return redirectTo(b"/gate?e=1", request)
+            return RedirectResponse("/gate/?e=1", 303)
 
         salt = bcrypt.gensalt()
         hashed = bcrypt.hashpw(passwd, salt)
 
-            result = self.data.user.create_user(
+        result = await self.data.user.create_user(
             uid, username, email.lower(), hashed.decode(), 1
         )
         if result is None:
-                return redirectTo(b"/gate?e=3", request)
+            return RedirectResponse("/gate/?e=3", 303)
 
-            if not self.data.user.check_password(uid, passwd):
+        if not await self.data.user.check_password(uid, passwd):
-                return redirectTo(b"/gate", request)
+            return RedirectResponse("/gate/", 303)
         
-            return redirectTo(b"/user", request)
+        sesh = UserSession()
+        sesh.user_id = uid
+        sesh.current_ip = ip
+        sesh.permissions = 1
         
-        else:
+        usr_sesh = self.encode_session(sesh)
-            return b""
+        self.logger.debug(f"Created session with JWT {usr_sesh}")
+        resp = RedirectResponse("/user/", 303)
+        resp.set_cookie("DIANA_SESH", usr_sesh)
 
-    def create_user(self, request: Request):
+        return resp
-        if b"ac" not in request.args or len(request.args[b"ac"][0].decode()) != 20:
-            return redirectTo(b"/gate?e=2", request)
 
-        ac = request.args[b"ac"][0].decode()
+    async def render_create_get(self, request: Request):
-        card = self.data.card.get_card_by_access_code(ac)
+        ac = request.query_params.get("ac", "")
+        if len(ac) != 20:
+            return RedirectResponse("/gate/?e=2", 303)
+
+        card = await self.data.card.get_card_by_access_code(ac)
         if card is None:
-            return redirectTo(b"/gate?e=1", request)
+            return RedirectResponse("/gate/?e=1", 303)
         
-        user = self.data.user.get_user(card['user'])
+        user = await self.data.user.get_user(card['user'])
         if user is None:
             self.logger.warning(f"Card {ac} exists with no/invalid associated user ID {card['user']}")
-            return redirectTo(b"/gate?e=0", request)
+            return RedirectResponse("/gate/?e=0", 303)
 
         if user['password'] is not None:
-            return redirectTo(b"/gate?e=1", request)
+            return RedirectResponse("/gate/?e=1", 303)
 
-        template = self.environment.get_template("core/frontend/gate/create.jinja")
+        template = self.environment.get_template("core/templates/gate/create.jinja")
-        return template.render(
+        return Response(template.render(
             title=f"{self.core_config.server.name} | Create User",
             code=ac,
-            sesh={"userId": 0, "permissions": 0},
+            sesh={"user_id": 0, "permissions": 0},
-        ).encode("utf-16")
+        ), media_type="text/html; charset=utf-8")
-
 
 class FE_User(FE_Base):
-    def render_GET(self, request: Request):
+    async def render_GET(self, request: Request):
-        uri = request.uri.decode()
+        uri = request.url.path
-        template = self.environment.get_template("core/frontend/user/index.jinja")
+        user_id = request.path_params.get('user_id', None)
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {uri}")
+        template = self.environment.get_template("core/templates/user/index.jinja")
 
-        sesh: Session = request.getSession()
+        usr_sesh = self.validate_session(request)
-        usr_sesh = IUserSession(sesh)
+        if not usr_sesh:
-        if usr_sesh.userId == 0:
+            return RedirectResponse("/gate/", 303)
-            return redirectTo(b"/gate", request)
         
-        m = re.match("\/user\/(\d*)", uri)
+        if user_id:
-        
+            if not self.test_perm(usr_sesh.permissions, PermissionOffset.USERMOD) and user_id != usr_sesh.user_id:
-        if m is not None:            
+                self.logger.warn(f"User {usr_sesh.user_id} does not have permission to view user {user_id}")
-            usrid = m.group(1)
+                return RedirectResponse("/user/", 303)
-            if usr_sesh.permissions < 1 << PermissionOffset.USERMOD.value or not usrid == usr_sesh.userId:
-                return redirectTo(b"/user", request)
         
         else:
-            usrid = usr_sesh.userId
+            user_id = usr_sesh.user_id
         
-        user = self.data.user.get_user(usrid)
+        user = await self.data.user.get_user(user_id)
         if user is None:
-            return redirectTo(b"/user", request)
+            self.logger.debug(f"User {user_id} not found")
+            return RedirectResponse("/user/", 303)
         
-        cards = self.data.card.get_user_cards(usrid)
+        cards = await self.data.card.get_user_cards(user_id)
-        arcades = self.data.arcade.get_arcades_managed_by_user(usrid)
         
         card_data = []
         arcade_data = []
@@ -285,176 +432,314 @@ class FE_User(FE_Base):
             else:
                 status = 'Active'
             
-            card_data.append({'access_code': c['access_code'], 'status': status})
+            #idm = c['idm']
+            ac = c['access_code']
 
-        for a in arcades:
+            if ac.startswith("5"): #or idm is not None:
-            arcade_data.append({'id': a['id'], 'name': a['name']})
+                c_type = "AmusementIC"
+            elif ac.startswith("3"):
+                c_type = "Banapass"
+            elif ac.startswith("010"):                
+                c_type = "Aime" # TODO: Aime verification
+            elif ac.startswith("0008"):
+                c_type = "Generated AIC"
+            else:
+                c_type = "Unknown"
             
-        return template.render(
+            card_data.append({
+                'access_code': ac, 
+                'status': status, 
+                'chip_id': "", #None if c['chip_id'] is None else f"{c['chip_id']:X}", 
+                'idm': "", 
+                'type': c_type, 
+                "memo": ""
+            })
+
+        if "e" in request.query_params:
+            try:
+                err = int(request.query_params.get("e", 0))
+            except Exception:
+                err = 0
+
+        else:
+            err = 0
+        
+        if "s" in request.query_params:
+            try:
+                succ = int(request.query_params.get("s", 0))
+            except Exception:
+                succ = 0
+
+        else:
+            succ = 0
+
+        return Response(template.render(
             title=f"{self.core_config.server.name} | Account", 
             sesh=vars(usr_sesh), 
             cards=card_data,
+            error=err,
+            success=succ,
             username=user['username'],
             arcades=arcade_data
-        ).encode("utf-16")
+        ), media_type="text/html; charset=utf-8")
     
-    def render_POST(self, request: Request):
+    async def render_logout(self, request: Request):
-        pass
+        resp = RedirectResponse("/gate/", 303)
+        resp.delete_cookie("DIANA_SESH")
+        return resp
     
+    async def edit_card(self, request: Request) -> RedirectResponse:
+        return RedirectResponse("/user/", 303)
+    
+    async def add_card(self, request: Request) -> RedirectResponse:
+        return RedirectResponse("/user/", 303)
+
+    async def render_POST(self, request: Request):
+        frm = await request.form()
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.USERMOD):
+            return RedirectResponse("/gate/", 303)
+        
+        old_pw: str = frm.get('current_pw', None)
+        pw1: str = frm.get('password1', None)
+        pw2: str = frm.get('password2', None)
+
+        if old_pw is None or pw1 is None or pw2 is None:
+            return RedirectResponse("/user/?e=4", 303)
+        
+        if pw1 != pw2:
+            return RedirectResponse("/user/?e=6", 303)
+        
+        if not await self.data.user.check_password(usr_sesh.user_id, old_pw.encode()):
+            return RedirectResponse("/user/?e=5", 303)
+        
+        if len(pw1) < 10 or not any(ele.isupper() for ele in pw1) or not any(ele.islower() for ele in pw1) \
+            or not any(ele.isdigit() for ele in pw1) or not any(not ele.isalnum() for ele in pw1):
+            return RedirectResponse("/user/?e=7", 303)
+        
+        salt = bcrypt.gensalt()
+        hashed = bcrypt.hashpw(pw1.encode(), salt)
+        if not await self.data.user.change_password(usr_sesh.user_id, hashed.decode()):
+            return RedirectResponse("/gate/?e=1", 303)
+        
+        return RedirectResponse("/user/?s=1", 303)
+        
+    async def update_username(self, request: Request):
+        frm = await request.form()
+        new_name: bytes = frm.get('new_name', "")
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.USERMOD):
+            return RedirectResponse("/gate/", 303)
+        
+        if new_name is None or not new_name:
+            return RedirectResponse("/user/?e=4", 303)
+        
+        if len(new_name) > 10:
+            return RedirectResponse("/user/?e=8", 303)
+
+        if not await self.data.user.change_username(usr_sesh.user_id, new_name):
+            return RedirectResponse("/user/?e=8", 303)
+        
+        return RedirectResponse("/user/?s=2", 303)
 
 class FE_System(FE_Base):
-    def render_GET(self, request: Request):
+    async def render_GET(self, request: Request):
-        uri = request.uri.decode()
+        template = self.environment.get_template("core/templates/sys/index.jinja")
-        template = self.environment.get_template("core/frontend/sys/index.jinja")
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.url.path}")
+
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm_minimum(usr_sesh.permissions, PermissionOffset.USERMOD):
+            return RedirectResponse("/gate/", 303)
+        
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | System", 
+            sesh=vars(usr_sesh), 
+            usrlist=[],
+        ), media_type="text/html; charset=utf-8")
+        
+    async def lookup_user(self, request: Request):
+        template = self.environment.get_template("core/templates/sys/index.jinja")
         usrlist: List[Dict] = []
-        aclist: List[Dict] = []
+        usr_sesh = self.validate_session(request)
-        cablist: List[Dict] = []
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.USERMOD):
+            return RedirectResponse("/gate/", 303)
         
-        sesh: Session = request.getSession()
+        uid_search = request.query_params.get("usrId",  None)
-        usr_sesh = IUserSession(sesh)
+        email_search = request.query_params.get("usrEmail", None)
-        if usr_sesh.userId == 0 or usr_sesh.permissions < 1 << PermissionOffset.USERMOD.value:
+        uname_search = request.query_params.get("usrName", None)
-            return redirectTo(b"/gate", request)
 
-        if uri.startswith("/sys/lookup.user?"):
+        if uid_search:
-            uri_parse = parse.parse_qs(uri.replace("/sys/lookup.user?", "")) # lop off the first bit
+            u = await self.data.user.get_user(uid_search)
-            uid_search = uri_parse.get("usrId")
-            email_search = uri_parse.get("usrEmail")
-            uname_search = uri_parse.get("usrName")
-
-            if uid_search is not None:
-                u = self.data.user.get_user(uid_search[0])
             if u is not None:
                 usrlist.append(u._asdict())
 
-            elif email_search is not None:
+        elif email_search:
-                u = self.data.user.find_user_by_email(email_search[0])
+            u = await self.data.user.find_user_by_email(email_search)
             if u is not None:
                 usrlist.append(u._asdict())
 
-            elif uname_search is not None:
+        elif uname_search:
-                ul = self.data.user.find_user_by_username(uname_search[0])
+            ul = await self.data.user.find_user_by_username(uname_search)
             for u in ul:
                 usrlist.append(u._asdict())
 
-        elif uri.startswith("/sys/lookup.arcade?"):
+        return Response(template.render(
-            uri_parse = parse.parse_qs(uri.replace("/sys/lookup.arcade?", "")) # lop off the first bit
-            ac_id_search = uri_parse.get("arcadeId")
-            ac_name_search = uri_parse.get("arcadeName")
-            ac_user_search = uri_parse.get("arcadeUser")
-            ac_ip_search = uri_parse.get("arcadeIp")
-
-            if ac_id_search is not None:
-                u = self.data.arcade.get_arcade(ac_id_search[0])
-                if u is not None:
-                    aclist.append(u._asdict())
-
-            elif ac_name_search is not None:
-                ul = self.data.arcade.get_arcade_by_name(ac_name_search[0])
-                if ul is not None:
-                    for u in ul:
-                        aclist.append(u._asdict())
-
-            elif ac_user_search is not None:
-                ul = self.data.arcade.get_arcades_managed_by_user(ac_user_search[0])
-                if ul is not None:
-                    for u in ul:
-                        aclist.append(u._asdict())
-
-            elif ac_ip_search is not None:
-                ul = self.data.arcade.get_arcades_by_ip(ac_ip_search[0])
-                if ul is not None:
-                    for u in ul:
-                        aclist.append(u._asdict())
-        
-        elif uri.startswith("/sys/lookup.cab?"):
-            uri_parse = parse.parse_qs(uri.replace("/sys/lookup.cab?", "")) # lop off the first bit
-            cab_id_search = uri_parse.get("cabId")
-            cab_serial_search = uri_parse.get("cabSerial")
-            cab_acid_search = uri_parse.get("cabAcId")
-
-            if cab_id_search is not None:
-                u = self.data.arcade.get_machine(id=cab_id_search[0])
-                if u is not None:
-                    cablist.append(u._asdict())
-
-            elif cab_serial_search is not None:
-                u = self.data.arcade.get_machine(serial=cab_serial_search[0])
-                if u is not None:
-                    cablist.append(u._asdict())
-
-            elif cab_acid_search is not None:
-                ul = self.data.arcade.get_arcade_machines(cab_acid_search[0])
-                for u in ul:
-                    cablist.append(u._asdict())
-
-        return template.render(
             title=f"{self.core_config.server.name} | System", 
             sesh=vars(usr_sesh), 
             usrlist=usrlist,
-            aclist=aclist,
+            shoplist=[],
-            cablist=cablist,
+        ), media_type="text/html; charset=utf-8")
-        ).encode("utf-16")
 
+    async def lookup_shop(self, request: Request):
+        shoplist = []
+        template = self.environment.get_template("core/templates/sys/index.jinja")
         
-class FE_Game(FE_Base):
+        usr_sesh = self.validate_session(request)
-    isLeaf = False
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.ACMOD):
-    children: Dict[str, Any] = {}
+            return RedirectResponse("/gate/", 303)
         
-    def getChild(self, name: bytes, request: Request):
+        shopid_search = request.query_params.get("shopId",  None)
-        if name == b"":
+        sn_search = request.query_params.get("serialNum", None)
-            return self
-        return resource.Resource.getChild(self, name, request)
         
-    def render_GET(self, request: Request) -> bytes:
+        if shopid_search:
-        return redirectTo(b"/user", request)
+            if shopid_search.isdigit():
-
+                shopid_search = int(shopid_search)
-
+                try:
-class FE_Arcade(FE_Base):
+                    sinfo = await self.data.arcade.get_arcade(shopid_search)
-    def render_GET(self, request: Request):
+                except Exception as e:
-        uri = request.uri.decode()
+                    self.logger.error(f"Failed to fetch shop info for shop {shopid_search} in lookup_shop - {e}")
-        template = self.environment.get_template("core/frontend/arcade/index.jinja")
+                    sinfo = None
-        managed = []
+                if sinfo:
-        
+                    shoplist.append({
-        sesh: Session = request.getSession()
+                        "name": sinfo['name'],
-        usr_sesh = IUserSession(sesh)
+                        "id": sinfo['id']
-        if usr_sesh.userId == 0:
+                    })
-            return redirectTo(b"/gate", request)
-        
-        m = re.match("\/arcade\/(\d*)", uri)
-        
-        if m is not None:
-            arcadeid = m.group(1)
-            perms = self.data.arcade.get_manager_permissions(usr_sesh.userId, arcadeid)
-            arcade = self.data.arcade.get_arcade(arcadeid)
-
-            if perms is None:
-                perms = 0
             
             else:
-            return redirectTo(b"/user", request)
+                return Response(template.render(
+                    title=f"{self.core_config.server.name} | System", 
+                    sesh=vars(usr_sesh),
+                    usrlist=[],
+                    shoplist=shoplist,
+                    error=4
+                ), media_type="text/html; charset=utf-8")
         
-        return template.render(
+        if sn_search:
+            sn_search = sn_search.upper().replace("-", "").strip()
+            if sn_search.isdigit() and len(sn_search) == 12:
+                prefix = sn_search[:4]
+                suffix = sn_search[5:]
+                
+                netid_prefix = self.environment.globals["sn_cvt"].get(prefix, "")                
+                sn_search = netid_prefix + suffix
+            
+            if re.match(r"^AB[DGL]N\d{7}$", sn_search) or re.match(r"^A\d{2}[EX]\d{2}[A-Z]\d{4,8}$", sn_search):
+                cabinfo = await self.data.arcade.get_machine(sn_search)
+                if cabinfo is None: sinfo = None
+                else:
+                    sinfo = await self.data.arcade.get_arcade(cabinfo['arcade'])
+                if sinfo:
+                    shoplist.append({
+                        "name": sinfo['name'],
+                        "id": sinfo['id']
+                    })
+            
+            else:
+                return Response(template.render(
+                    title=f"{self.core_config.server.name} | System", 
+                    sesh=vars(usr_sesh),
+                    usrlist=[],
+                    shoplist=shoplist,
+                    error=10
+                ), media_type="text/html; charset=utf-8")
+            
+        
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | System", 
+            sesh=vars(usr_sesh), 
+            usrlist=[],
+            shoplist=shoplist,
+        ), media_type="text/html; charset=utf-8")
+
+class FE_Arcade(FE_Base):
+    async def render_GET(self, request: Request):
+        template = self.environment.get_template("core/templates/arcade/index.jinja")
+        shop_id = request.path_params.get('shop_id', None)
+        
+        usr_sesh = self.validate_session(request)
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.ACMOD):
+            self.logger.warn(f"User {usr_sesh.user_id} does not have permission to view shops!")
+            return RedirectResponse("/gate/", 303)
+        
+        if not shop_id:
+            return Response(template.render(
                 title=f"{self.core_config.server.name} | Arcade", 
                 sesh=vars(usr_sesh),
-            error=0,
+            ), media_type="text/html; charset=utf-8")
-            perms=perms,
-            arcade=arcade._asdict()
-        ).encode("utf-16")
         
+        sinfo = await self.data.arcade.get_arcade(shop_id)
+        if not sinfo:
+            return Response(template.render(
+                title=f"{self.core_config.server.name} | Arcade", 
+                sesh=vars(usr_sesh),
+            ), media_type="text/html; charset=utf-8")
+        
+        cabs = await self.data.arcade.get_arcade_machines(shop_id)
+        cablst = []
+        if cabs:
+            for x in cabs:
+                cablst.append({
+                    "id": x['id'],
+                    "serial": x['serial'],
+                    "game": x['game'],
+                })
+        
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | Arcade", 
+            sesh=vars(usr_sesh),
+            arcade={
+                "name": sinfo['name'],
+                "id": sinfo['id'],
+                "cabs": cablst
+            }
+            
+        ), media_type="text/html; charset=utf-8")
 
 class FE_Machine(FE_Base):
-    def render_GET(self, request: Request):
+    async def render_GET(self, request: Request):
-        uri = request.uri.decode()
+        template = self.environment.get_template("core/templates/machine/index.jinja")
-        template = self.environment.get_template("core/frontend/machine/index.jinja")
+        cab_id = request.path_params.get('cab_id', None)
         
-        sesh: Session = request.getSession()
+        usr_sesh = self.validate_session(request)
-        usr_sesh = IUserSession(sesh)
+        if not usr_sesh or not self.test_perm(usr_sesh.permissions, PermissionOffset.ACMOD):
-        if usr_sesh.userId == 0:
+            self.logger.warn(f"User {usr_sesh.user_id} does not have permission to view shops!")
-            return redirectTo(b"/gate", request)
+            return RedirectResponse("/gate/", 303)
         
-        return template.render(
+        if not cab_id:
+            return Response(template.render(
                 title=f"{self.core_config.server.name} | Machine", 
                 sesh=vars(usr_sesh),
-            arcade={},
+            ), media_type="text/html; charset=utf-8")
-            error=0,
+        
-        ).encode("utf-16")
+        return Response(template.render(
+            title=f"{self.core_config.server.name} | Machine", 
+            sesh=vars(usr_sesh),
+            arcade={}
+        ), media_type="text/html; charset=utf-8")
+
+cfg_dir = environ.get("DIANA_CFG_DIR", "config")
+cfg: CoreConfig = CoreConfig()
+if path.exists(f"{cfg_dir}/core.yaml"):
+    cfg.update(yaml.safe_load(open(f"{cfg_dir}/core.yaml")))
+
+if not path.exists(cfg.server.log_dir):
+    mkdir(cfg.server.log_dir)
+
+if not access(cfg.server.log_dir, W_OK):
+    print(
+        f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+    )
+    exit(1)
+
+fe = FrontendServlet(cfg, cfg_dir)
+app = Starlette(cfg.server.is_develop, fe.get_routes(), on_startup=[fe.startup])

core/frontend/arcade/index.jinja

@@ -1,4 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-<h1>{{ arcade.name }}</h1>
-{% endblock content %}

core/frontend/machine/index.jinja

@@ -1,5 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-{% include "core/frontend/widgets/err_banner.jinja" %}
-<h1>Machine Management</h1>
-{% endblock content %}

core/frontend/sys/index.jinja

@@ -1,103 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-<h1>System Management</h1>
-
-<div class="row" id="rowForm">
-    {% if sesh.permissions >= 2 %}
-    <div class="col-sm-6" style="max-width: 25%;">
-        <form id="usrLookup" name="usrLookup" action="/sys/lookup.user" class="form-inline">
-            <h3>User Search</h3>
-            <div class="form-group">
-                <label for="usrId">User ID</label>
-                <input type="number" class="form-control" id="usrId" name="usrId">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="usrName">Username</label>
-                <input type="text" class="form-control" id="usrName" name="usrName">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="usrEmail">Email address</label>
-                <input type="email" class="form-control" id="usrEmail" name="usrEmail" aria-describedby="emailHelp">
-            </div>
-            <br />
-            <button type="submit" class="btn btn-primary">Search</button>
-        </form>
-    </div>
-    {% endif %}
-    {% if sesh.permissions >= 4 %}
-    <div class="col-sm-6" style="max-width: 25%;">
-        <form id="arcadeLookup" name="arcadeLookup" action="/sys/lookup.arcade" class="form-inline" >
-            <h3>Arcade Search</h3>
-            <div class="form-group">
-                <label for="arcadeId">Arcade ID</label>
-                <input type="number" class="form-control" id="arcadeId" name="arcadeId">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="arcadeName">Arcade Name</label>
-                <input type="text" class="form-control" id="arcadeName" name="arcadeName">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="arcadeUser">Owner User ID</label>
-                <input type="number" class="form-control" id="arcadeUser" name="arcadeUser">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="arcadeIp">Assigned IP Address</label>
-                <input type="text" class="form-control" id="arcadeIp" name="arcadeIp">
-            </div>
-            <br />
-            <button type="submit" class="btn btn-primary">Search</button>
-        </form>
-    </div>
-    <div class="col-sm-6" style="max-width: 25%;">
-        <form id="cabLookup" name="cabLookup" action="/sys/lookup.cab" class="form-inline" >
-            <h3>Machine Search</h3>
-            <div class="form-group">
-                <label for="cabId">Machine ID</label>
-                <input type="number" class="form-control" id="cabId" name="cabId">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="cabSerial">Machine Serial</label>
-                <input type="text" class="form-control" id="cabSerial" name="cabSerial">
-            </div>
-            OR
-            <div class="form-group">
-                <label for="cabAcId">Arcade ID</label>
-                <input type="number" class="form-control" id="cabAcId" name="cabAcId">
-            </div>
-            <br />
-            <button type="submit" class="btn btn-primary">Search</button>
-        </form>
-    </div>
-    {% endif %}
-</div>
-<div class="row" id="rowResult" style="margin: 10px;">
-    {% if sesh.permissions >= 2 %}
-    <div id="userSearchResult" class="col-sm-6" style="max-width: 25%;">
-        {% for usr in usrlist %}
-        <a href=/user/{{ usr.id }}><pre>{{ usr.id }} | {{ usr.username if usr.username != None else "<i>No Name Set</i>"}}</pre></a>
-        {% endfor %}
-    </div>
-    {% endif %}
-    {% if sesh.permissions >= 4 %}
-    <div id="arcadeSearchResult" class="col-sm-6" style="max-width: 25%;">
-        {% for ac in aclist %}
-        <pre><a href=/arcade/{{ ac.id }}>{{ ac.id }} | {{ ac.name if ac.name != None else "<i>No Name Set</i>" }} | {{ ac.ip if ac.ip != None else "<i>No IP Assigned</i>"}}</pre></a>
-        {% endfor %}
-    </div
-    ><div id="cabSearchResult" class="col-sm-6" style="max-width: 25%;">
-        {% for cab in cablist %}
-        <a href=/cab/{{ cab.id }}><pre>{{ cab.id }} | {{ cab.game if cab.game != None else "<i>ANY </i>" }} | {{ cab.serial }}</pre></a>
-        {% endfor %}
-    </div>
-    {% endif %}
-</div>
-<div class="row" id="rowAdd">
-
-</div>
-{% endblock content %}

core/frontend/user/index.jinja

@@ -1,41 +0,0 @@
-{% extends "core/frontend/index.jinja" %}
-{% block content %}
-<h1>Management for {{ username }}</h1>
-<h2>Cards <button class="btn btn-success" data-bs-toggle="modal" data-bs-target="#card_add">Add</button></h2>
-<ul style="font-size: 20px;">
-{% for c in cards %}
-<li>{{ c.access_code }}: {{ c.status }}&nbsp;{% if c.status == 'Active'%}<button class="btn-warning btn">Lock</button>{% elif c.status == 'Locked' %}<button class="btn-warning btn">Unlock</button>{% endif %}&nbsp;<button class="btn-danger btn">Delete</button></li>
-{% endfor %}
-</ul>
-
-{% if arcades is defined %}
-<h2>Arcades</h2>
-  <ul style="font-size: 20px;">
-  {% for a in arcades %}
-    <li><a href=/arcade/{{ a.id }}>{{ a.name }}</a></li>
-  {% endfor %}
-  </ul>
-{% endif %}
-
-<div class="modal fade" id="card_add" tabindex="-1" aria-labelledby="card_add_label" aria-hidden="true">
-  <div class="modal-dialog">
-    <div class="modal-content">
-      <div class="modal-header">
-        <h1 class="modal-title fs-5" id="card_add_label">Add Card</h1>
-        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
-      </div>
-      <div class="modal-body">
-        HOW TO:<br>
-        Scan your card on any networked game and press the "View Access Code" button (varies by game) and enter the 20 digit code below.<br>
-        !!FOR AMUSEIC CARDS: DO NOT ENTER THE CODE SHOWN ON THE BACK OF THE CARD ITSELF OR IT WILL NOT WORK!!
-        <p /><label for="card_add_frm_access_code">Access Code:&nbsp;</label><input id="card_add_frm_access_code" maxlength="20" type="text" required> 
-      </div>
-      <div class="modal-footer">
-        <button type="button" class="btn btn-primary">Add</button>
-        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
-      </div>
-    </div>
-  </div>
-</div>
-
-{% endblock content %}

core/mucha.py

@@ -1,8 +1,8 @@
-from typing import Dict, Any, Optional, List
+from typing import Dict, Any, Optional
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
-from twisted.web import resource
+from starlette.requests import Request
-from twisted.web.http import Request
+from starlette.responses import PlainTextResponse
 from datetime import datetime
 from Crypto.Cipher import Blowfish
 import pytz
@@ -10,9 +10,11 @@ import pytz
 from .config import CoreConfig
 from .utils import Utils
 from .title import TitleServlet
+from .data import Data
+from .const import *
 
 class MuchaServlet:
-    mucha_registry: List[str] = []
+    mucha_registry: Dict[str, Dict[str, str]] = {}
     def __init__(self, cfg: CoreConfig, cfg_dir: str) -> None:
         self.config = cfg
         self.config_dir = cfg_dir
@@ -37,89 +39,147 @@ class MuchaServlet:
         self.logger.setLevel(cfg.mucha.loglevel)
         coloredlogs.install(level=cfg.mucha.loglevel, logger=self.logger, fmt=log_fmt_str)
         
+        self.data = Data(cfg)
+
         for _, mod in TitleServlet.title_registry.items():
-            if hasattr(mod, "get_mucha_info"):
+            enabled, game_cds, netids = mod.get_mucha_info(self.config, self.config_dir)
-                enabled, game_cd = mod.get_mucha_info(
-                    self.config, self.config_dir
-                )
             if enabled:
-                    self.mucha_registry.append(game_cd)
+                for x in range(len(game_cds)):
+                    self.mucha_registry[game_cds[x]] = { "netid_prefix": netids[x] }
 
         self.logger.info(f"Serving {len(self.mucha_registry)} games")
 
-    def handle_boardauth(self, request: Request, _: Dict) -> bytes:
+    async def handle_boardauth(self, request: Request) -> bytes:
-        req_dict = self.mucha_preprocess(request.content.getvalue())
+        bod = await request.body()
+        req_dict = self.mucha_preprocess(bod)
         client_ip = Utils.get_ip_addr(request)
 
         if req_dict is None:
             self.logger.error(
-                f"Error processing mucha request {request.content.getvalue()}"
+                f"Error processing mucha request {bod}"
             )
-            return b"RESULTS=000"
+            return PlainTextResponse("RESULTS=000")
 
         req = MuchaAuthRequest(req_dict)
-        self.logger.info(f"Boardauth request from {client_ip} for {req.gameVer}")        
         self.logger.debug(f"Mucha request {vars(req)}")
         
-        if req.gameCd not in self.mucha_registry:
+        if not req.gameCd or not req.gameVer or not req.sendDate or not req.countryCd or not req.serialNum:
-            self.logger.warning(f"Unknown gameCd {req.gameCd}")
+            self.logger.warn(f"Missing required fields - {vars(req)}")
-            return b"RESULTS=000"
+            return PlainTextResponse("RESULTS=000")
+
+        minfo = self.mucha_registry.get(req.gameCd, {})
+
+        if not minfo:
+            self.logger.warning(f"Unknown gameCd {req.gameCd} from {client_ip}")
+            return PlainTextResponse("RESULTS=000")
 
-        # TODO: Decrypt S/N
         b_key = b""
         for x in range(8):
             b_key += req.sendDate[(x - 1) & 7].encode()
         
-        cipher = Blowfish.new(b_key, Blowfish.MODE_ECB)
+        b_iv = b_key # what the fuck namco
-        sn_decrypt = cipher.decrypt(bytes.fromhex(req.serialNum))
+
-        self.logger.debug(f"Decrypt SN to {sn_decrypt.hex()}")
+        cipher = Blowfish.new(b_key, Blowfish.MODE_CBC, b_iv)
+        try:
+            sn_decrypt = cipher.decrypt(bytes.fromhex(req.serialNum))[:12].decode()
+        except Exception as e:
+            self.logger.error(f"Decrypt SN {req.serialNum} failed! - {e}")
+            return PlainTextResponse("RESULTS=000")
+
+        self.logger.info(f"Boardauth request from {sn_decrypt} ({client_ip}) for {req.gameVer}")
 
         resp = MuchaAuthResponse(
-            f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}"
+            f"{self.config.server.hostname}{':' + str(self.config.server.port) if not self.config.server.is_using_proxy else ''}"
         )
 
+        netid = minfo.get('netid_prefix', "ABxN") + sn_decrypt[5:]
+
+        cab = await self.data.arcade.get_machine(netid)
+        if cab:
+            arcade = await self.data.arcade.get_arcade(cab['id'])
+            if not arcade:
+                self.logger.error(f"Failed to get arcade with id {cab['id']}")
+                return PlainTextResponse("RESULTS=000")
+
+            resp.AREA_0 = arcade["region_id"] or AllnetJapanRegionId.AICHI.name
+            resp.AREA_0_EN = arcade["region_id"] or AllnetJapanRegionId.AICHI.name
+            resp.AREA_FULL_0 = arcade["region_id"] or AllnetJapanRegionId.AICHI.name
+            resp.AREA_FULL_0_EN = arcade["region_id"] or AllnetJapanRegionId.AICHI.name
+            
+            resp.AREA_1 = arcade["country"] or cab['country'] or AllnetCountryCode.JAPAN.value
+            resp.AREA_1_EN = arcade["country"] or cab['country'] or AllnetCountryCode.JAPAN.value
+            resp.AREA_FULL_1 = arcade["country"] or cab['country'] or AllnetCountryCode.JAPAN.value
+            resp.AREA_FULL_1_EN = arcade["country"] or cab['country'] or AllnetCountryCode.JAPAN.value
+
+            resp.AREA_2 = arcade["city"] if arcade["city"] else ""
+            resp.AREA_2_EN = arcade["city"] if arcade["city"] else ""
+            resp.AREA_FULL_2 = arcade["city"] if arcade["city"] else ""
+            resp.AREA_FULL_2_EN = arcade["city"] if arcade["city"] else ""
+
+            resp.AREA_3 = ""
+            resp.AREA_3_EN = ""
+            resp.AREA_FULL_3 = ""
+            resp.AREA_FULL_3_EN = ""
+
+            resp.PREFECTURE_ID = arcade['region_id']
+            resp.COUNTRY_CD = arcade['country'] or cab['country'] or AllnetCountryCode.JAPAN.value
+            resp.PLACE_ID = req.placeId if req.placeId else f"{arcade['country'] or cab['country'] or AllnetCountryCode.JAPAN.value}{arcade['id']:04X}"
+            resp.SHOP_NAME = arcade['name']
+            resp.SHOP_NAME_EN = arcade['name']
+            resp.SHOP_NICKNAME = arcade['nickname']
+            resp.SHOP_NICKNAME_EN = arcade['nickname']
+
+        elif self.config.server.allow_unregistered_serials:
+            self.logger.info(f"Allow unknown serial {netid} ({sn_decrypt}) to auth")
+        
+        else:
+            self.logger.warn(f'Auth failed for NetID {netid}')
+            return PlainTextResponse("RESULTS=000")
+
         self.logger.debug(f"Mucha response {vars(resp)}")
 
-        return self.mucha_postprocess(vars(resp))
+        return PlainTextResponse(self.mucha_postprocess(vars(resp)))
 
-    def handle_updatecheck(self, request: Request, _: Dict) -> bytes:
+    async def handle_updatecheck(self, request: Request) -> bytes:
-        req_dict = self.mucha_preprocess(request.content.getvalue())
+        bod = await request.body()
+        req_dict = self.mucha_preprocess(bod)
         client_ip = Utils.get_ip_addr(request)
 
         if req_dict is None:
             self.logger.error(
-                f"Error processing mucha request {request.content.getvalue()}"
+                f"Error processing mucha request {bod}"
             )
-            return b"RESULTS=000"
+            return PlainTextResponse("RESULTS=000")
 
         req = MuchaUpdateRequest(req_dict)
-        self.logger.info(f"Updatecheck request from {client_ip} for {req.gameVer}")        
+        self.logger.info(f"Updatecheck request from {req.serialNum} ({client_ip}) for {req.gameVer}")        
         self.logger.debug(f"Mucha request {vars(req)}")
 
         if req.gameCd not in self.mucha_registry:
             self.logger.warning(f"Unknown gameCd {req.gameCd}")
-            return b"RESULTS=000"
+            return PlainTextResponse("RESULTS=000")
 
-        resp = MuchaUpdateResponse(req.gameVer, f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}")
+        resp = MuchaUpdateResponse(req.gameVer, f"{self.config.server.hostname}{':' + str(self.config.server.port) if not self.config.server.is_using_proxy else ''}")
 
         self.logger.debug(f"Mucha response {vars(resp)}")
 
-        return self.mucha_postprocess(vars(resp))
+        return PlainTextResponse(self.mucha_postprocess(vars(resp)))
 
-    def handle_dlstate(self, request: Request, _: Dict) -> bytes:
+    async def handle_dlstate(self, request: Request) -> bytes:
-        req_dict = self.mucha_preprocess(request.content.getvalue())
+        bod = await request.body()
+        req_dict = self.mucha_preprocess(bod)
         client_ip = Utils.get_ip_addr(request)
 
         if req_dict is None:
             self.logger.error(
-                f"Error processing mucha request {request.content.getvalue()}"
+                f"Error processing mucha request {bod}"
             )
-            return b""
+            return PlainTextResponse("RESULTS=000")
         
         req = MuchaDownloadStateRequest(req_dict)
-        self.logger.info(f"DownloadState request from {client_ip} for {req.gameCd} -> {req.updateVer}")        
+        self.logger.info(f"DownloadState request from {req.serialNum} ({client_ip}) for {req.gameCd} -> {req.updateVer}")        
         self.logger.debug(f"request {vars(req)}")
-        return b"RESULTS=001"
+        return PlainTextResponse("RESULTS=001")
 
     def mucha_preprocess(self, data: bytes) -> Optional[Dict]:
         try:
@@ -169,7 +229,7 @@ class MuchaAuthResponse:
         self.RESULTS = "001"
         self.AUTH_INTERVAL = "86400"
         self.SERVER_TIME = datetime.strftime(datetime.now(), "%Y%m%d%H%M")
-        self.UTC_SERVER_TIME = datetime.strftime(datetime.now(pytz.UTC), "%Y%m%d%H%M")
+        self.SERVER_TIME_UTC = datetime.strftime(datetime.now(pytz.UTC), "%Y%m%d%H%M")
 
         self.CHARGE_URL = f"https://{mucha_url}/charge/"
         self.FILE_URL = f"https://{mucha_url}/file/"

core/templates/arcade/index.jinja

@@ -0,0 +1,19 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+{% if arcade is defined %}
+<h1>{{ arcade.name }}</h1>
+<h2>PCBs assigned to this arcade <button class="btn btn-success" id="btn_add_cab" onclick="toggle_add_cab_form()">Add</button></h2>
+{% if success is defined and success == 3 %}
+<div style="background-color: #00AA00; padding: 20px; margin-bottom: 10px; width: 15%;">
+Cab added successfully
+</div>
+{% endif %}
+<ul style="font-size: 20px;">
+{% for c in arcade.cabs %}
+<li><a href="/cab/{{ c.id }}">{{ c.serial }} ({{ c.game }})</a>&nbsp;<button class="btn btn-secondary" onclick="prep_edit_form()">Edit</button>&nbsp;<button class="btn-danger btn">Delete</button></li>
+{% endfor %}
+</ul>
+{% else %}
+<h3>Arcade Not Found</h3>
+{% endif %}
+{% endblock content %}

core/templates/gate/create.jinja

@@ -1,4 +1,4 @@
-{% extends "core/frontend/index.jinja" %}
+{% extends "core/templates/index.jinja" %}
 {% block content %}
 <h1>Create User</h1>
 <form id="create" style="max-width: 240px; min-width: 10%;" action="/gate/gate.create" method="post">

core/templates/gate/gate.jinja

@@ -1,7 +1,7 @@
-{% extends "core/frontend/index.jinja" %}
+{% extends "core/templates/index.jinja" %}
 {% block content %}
 <h1>Gate</h1>
-{% include "core/frontend/widgets/err_banner.jinja" %}
+{% include "core/templates/widgets/err_banner.jinja" %}
 <style>
     /* Chrome, Safari, Edge, Opera */
     input::-webkit-outer-spin-button,

core/templates/index.jinja

@@ -84,7 +84,7 @@
         </style>
     </head>
     <body>
-    {% include "core/frontend/widgets/topbar.jinja" %}
+    {% include "core/templates/widgets/topbar.jinja" %}
     {% block content %}
     <h1>{{ server_name }}</h1>
     {% endblock content %}

core/templates/machine/index.jinja

@@ -0,0 +1,4 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+<h1>Machine Management</h1>
+{% endblock content %}

core/templates/sys/index.jinja

@@ -0,0 +1,69 @@
+{% extends "core/templates/index.jinja" %}
+{% block content %}
+<h1>System Management</h1>
+{% if error is defined %}
+{% include "core/templates/widgets/err_banner.jinja" %}
+{% endif %}
+<div class="row" id="rowForm">
+    {% if "{:08b}".format(sesh.permissions)[6] == "1" %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="usrLookup" name="usrLookup" action="/sys/lookup.user" class="form-inline">
+            <h3>User Search</h3>
+            <div class="form-group">
+                <label for="usrId">User ID</label>
+                <input type="number" class="form-control" id="usrId" name="usrId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrName">Username</label>
+                <input type="text" class="form-control" id="usrName" name="usrName">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="usrEmail">Email address</label>
+                <input type="email" class="form-control" id="usrEmail" name="usrEmail" aria-describedby="emailHelp">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    {% endif %}
+    {% if "{:08b}".format(sesh.permissions)[5] == "1" %}
+    <div class="col-sm-6" style="max-width: 25%;">
+        <form id="shopLookup" name="shopLookup" action="/sys/lookup.shop" class="form-inline">
+            <h3>Shop search</h3>
+            <div class="form-group">
+                <label for="shopId">Shop ID</label>
+                <input type="number" class="form-control" id="shopId" name="shopId">
+            </div>
+            OR
+            <div class="form-group">
+                <label for="serialNum">Serial Number</label>
+                <input type="text" class="form-control" id="serialNum" name="serialNum" maxlength="15">
+            </div>
+            <br />
+            <button type="submit" class="btn btn-primary">Search</button>
+        </form>
+    </div>
+    {% endif %}
+</div>
+<div class="row" id="rowResult" style="margin: 10px;">
+    {% if "{:08b}".format(sesh.permissions)[6] == "1" %}
+    <div id="userSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for usr in usrlist %}
+        <a href=/user/{{ usr.id }}><pre>{{ usr.username if usr.username is not none else "<i>No Name Set</i>"}}</pre></a>
+        {% endfor %}
+    </div>
+    {% endif %}
+    {% if "{:08b}".format(sesh.permissions)[5] == "1" %}
+    <div id="shopSearchResult" class="col-sm-6" style="max-width: 25%;">
+        {% for shop in shoplist %}
+        <a href="/shop/{{ shop.id }}"><pre>{{ shop.name if shop.name else "<i>No Name Set</i>"}}</pre></a>
+        {% endfor %}
+    </div>
+    {% endif %}
+</div>
+<div class="row" id="rowAdd">
+
+</div>
+{% endblock content %}

core/templates/widgets/err_banner.jinja

@@ -8,9 +8,19 @@ Missing or malformed access code
 {% elif error == 3 %}
 Failed to create user
 {% elif error == 4 %}
-Arcade not found
+Required field not filled or invalid
 {% elif error == 5 %}
-Machine not found
+Incorrect old password
+{% elif error == 6 %}
+Passwords don't match
+{% elif error == 7 %}
+New password not acceptable
+{% elif error == 8 %}
+New Nickname too long
+{% elif error == 9 %}
+You must be logged in to preform this action
+{% elif error == 10 %}
+Invalid serial number
 {% else %}
 An unknown error occoured
 {% endif %}

core/templates/widgets/topbar.jinja

@@ -3,19 +3,20 @@
 </div>
 <div style="background: #333; color: #f9f9f9; width: 80%; height: 50px; line-height: 50px; padding-left: 10px; float: left;">
     <a href=/><button class="btn btn-primary">Home</button></a>&nbsp;
-    {% for game in game_list %}    
+    {% for game, data in game_list|items %}    
-    <a href=/game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
+    <a href=/game{{ data.url }}/><button class="btn btn-success">{{ game }}</button></a>&nbsp;
     {% endfor %}
 </div>
 </div>
 <div style="background: #333; color: #f9f9f9; width: 10%; height: 50px; line-height: 50px; text-align: center; float: left;">
     {% if sesh is defined and sesh["permissions"] >= 2 %}
-    <a href="/sys"><button class="btn btn-primary">System</button></a>
+    <a href="/sys/"><button class="btn btn-primary">System</button></a>
     {% endif %}
-    {% if sesh is defined and sesh["userId"] > 0 %}
+    {% if sesh is defined and sesh["user_id"] > 0 %}
-    <a href="/user"><button class="btn btn-primary">Account</button></a>
+    <a href="/user/"><button class="btn btn-primary">Account</button></a>
+    <a href="/user/logout"><button class="btn btn-danger">Logout</button></a>
     {% else %}
-    <a href="/gate"><button class="btn btn-primary">Gate</button></a>
+    <a href="/gate/"><button class="btn btn-primary">Gate</button></a>
     {% endif %}
     
 </div>

core/title.py

@@ -1,12 +1,24 @@
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Any
+import json
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
-from twisted.web.http import Request
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Route
 
 from core.config import CoreConfig
 from core.data import Data
 from core.utils import Utils
 
+class JSONResponseNoASCII(Response):
+    media_type = "application/json"
+
+    def render(self, content: Any) -> bytes:
+        return json.dumps(
+            content,
+            ensure_ascii=False,
+        ).encode("utf-8")
+
 class BaseServlet:
     def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
         self.core_cfg = core_cfg
@@ -28,18 +40,16 @@ class BaseServlet:
         """
         return False
     
-    def get_endpoint_matchers(self) -> Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]:
+    def get_routes(self) -> List[Route]:
         """Called during boot to get all matcher endpoints this title servlet handles
 
         Returns:
-            Tuple[List[Tuple[str, str, Dict]], List[Tuple[str, str, Dict]]]: A 2-length tuple where offset 0 is GET and offset 1 is POST, 
+            List[Route]: A list of Routes, WebSocketRoutes, or similar classes
-            containing a list of 3-length tuples where offset 0 is the name of the function in the handler that should be called, offset 1
-            is the matching string, and offset 2 is a dict containing rules for the matcher.
         """
-        return (
+        return [
-            [("render_GET", "/{game}/{version}/{endpoint}", {'game': R'S...'})], 
+            Route("/{game}/{version}/{endpoint}", self.render_POST, methods=["POST"]),
-            [("render_POST", "/{game}/{version}/{endpoint}", {'game': R'S...'})]
+            Route("/{game}/{version}/{endpoint}", self.render_GET, methods=["GET"]),
-        )
+        ]
     
     def setup(self) -> None:
         """Called once during boot, should contain any additional setup the handler must do, such as starting any sub-services
@@ -58,11 +68,11 @@ class BaseServlet:
             Tuple[str, str]: A tuple where offset 0 is the allnet uri field, and offset 1 is the allnet host field
         """
         if not self.core_cfg.server.is_using_proxy and Utils.get_title_port(self.core_cfg) != 80:
-            return (f"http://{self.core_cfg.title.hostname}:{Utils.get_title_port(self.core_cfg)}/{game_code}/{game_ver}/", "")
+            return (f"http://{self.core_cfg.server.hostname}:{Utils.get_title_port(self.core_cfg)}/{game_code}/{game_ver}/", "")
 
-        return (f"http://{self.core_cfg.title.hostname}/{game_code}/{game_ver}/", "")
+        return (f"http://{self.core_cfg.server.hostname}/{game_code}/{game_ver}/", "")
 
-    def get_mucha_info(self, core_cfg: CoreConfig, cfg_dir: str) -> Tuple[bool, str]:
+    def get_mucha_info(self, core_cfg: CoreConfig, cfg_dir: str) -> Tuple[bool, List[str], List[str]]:
         """Called once during boot to check if this game is a mucha game
 
         Args:
@@ -72,15 +82,15 @@ class BaseServlet:
         Returns:
             Tuple[bool, str]: Tuple where offset 0 is true if the game is enabled, false otherwise, and offset 1 is the game CD
         """
-        return (False, "")
+        return (False, [], [])
 
-    def render_POST(self, request: Request, game_code: str, matchers: Dict) -> bytes:
+    async def render_POST(self, request: Request) -> bytes:
-        self.logger.warn(f"{game_code} Does not dispatch POST")
+        self.logger.warn(f"Game Does not dispatch POST")
-        return None
+        return Response()
 
-    def render_GET(self, request: Request, game_code: str, matchers: Dict) -> bytes:
+    async def render_GET(self, request: Request) -> bytes:
-        self.logger.warn(f"{game_code} Does not dispatch GET")
+        self.logger.warn(f"Game Does not dispatch GET")
-        return None
+        return Response()
 
 class TitleServlet:
     title_registry: Dict[str, BaseServlet] = {}
@@ -136,7 +146,7 @@ class TitleServlet:
                 self.logger.error(f"{folder} missing game_code or index in __init__.py, or is_game_enabled in index")
 
         self.logger.info(
-            f"Serving {len(self.title_registry)} game codes {'on port ' + str(core_cfg.title.port) if core_cfg.title.port > 0 else ''}"
+            f"Serving {len(self.title_registry)} game codes {'on port ' + str(core_cfg.server.port) if core_cfg.server.port > 0 else ''}"
         )
 
     def render_GET(self, request: Request, endpoints: dict) -> bytes:

core/utils.py

@@ -1,6 +1,6 @@
 from typing import Dict, Any, Optional
 from types import ModuleType
-from twisted.web.http import Request
+from starlette.requests import Request
 import logging
 import importlib
 from os import walk
@@ -34,21 +34,13 @@ class Utils:
 
     @classmethod
     def get_ip_addr(cls, req: Request) -> str:
-        return (
+        return req.headers.get("x-forwarded-for", req.client.host)
-            req.getAllHeaders()[b"x-forwarded-for"].decode()
-            if b"x-forwarded-for" in req.getAllHeaders()
-            else req.getClientAddress().host
-        )
     
     @classmethod
     def get_title_port(cls, cfg: CoreConfig):
         if cls.real_title_port is not None: return cls.real_title_port
 
-        if cfg.title.port == 0:
+        cls.real_title_port = cfg.server.proxy_port if cfg.server.is_using_proxy and cfg.server.proxy_port else cfg.server.port
-            cls.real_title_port = cfg.allnet.port
-        
-        else:
-            cls.real_title_port = cfg.title.port
         
         return cls.real_title_port
     
@@ -56,11 +48,7 @@ class Utils:
     def get_title_port_ssl(cls, cfg: CoreConfig):
         if cls.real_title_port_ssl is not None: return cls.real_title_port_ssl
 
-        if cfg.title.port_ssl == 0:
+        cls.real_title_port_ssl = cfg.server.proxy_port_ssl if cfg.server.is_using_proxy and cfg.server.proxy_port_ssl else Utils.get_title_port(cfg)
-            cls.real_title_port_ssl = 443
-        
-        else:
-            cls.real_title_port_ssl = cfg.title.port_ssl
         
         return cls.real_title_port_ssl
 

dbutils.py

@@ -1,9 +1,12 @@
-import yaml
+#!/usr/bin/env python3
 import argparse
 import logging
-from core.config import CoreConfig
+from os import mkdir, path, access, W_OK
+import yaml
+import asyncio
+
 from core.data import Data
-from os import path, mkdir, access, W_OK
+from core.config import CoreConfig
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Database utilities")
@@ -16,19 +19,10 @@ if __name__ == "__main__":
         type=str,
         help="Version of the database to upgrade/rollback to",
     )
-    parser.add_argument(
-        "--game",
-        "-g",
-        type=str,
-        help="Game code of the game who's schema will be updated/rolled back. Ex. SDFE",
-    )
     parser.add_argument("--email", "-e", type=str, help="Email for the new user")
-    parser.add_argument("--old_ac", "-o", type=str, help="Access code to transfer from")
+    parser.add_argument("--access_code", "-a", type=str, help="Access code for new/transfer user", default="00000000000000000000")
-    parser.add_argument("--new_ac", "-n", type=str, help="Access code to transfer to")
+    parser.add_argument("--message", "-m", type=str, help="Revision message")
-    parser.add_argument("--force", "-f", type=bool, help="Force the action to happen")
+    parser.add_argument("action", type=str, help="create, upgrade, downgrade, create-owner, migrate, create-revision, create-autorevision")
-    parser.add_argument(
-        "action", type=str, help="DB Action, create, recreate, upgrade, or rollback"
-    )
     args = parser.parse_args()
 
     cfg = CoreConfig()
@@ -51,41 +45,30 @@ if __name__ == "__main__":
     if args.action == "create":
         data.create_database()
     
-    elif args.action == "recreate":
+    elif args.action == "upgrade":
-        data.recreate_database()
+        data.schema_upgrade(args.version)
 
-    elif args.action == "upgrade" or args.action == "rollback":
+    elif args.action == "downgrade":
-        if args.version is None:
+        if not args.version:
-            data.logger.warning("No version set, upgrading to latest")
+            logging.getLogger("database").error(f"Version argument required for downgrade")
-
+            exit(1)
-        if args.game is None:
+        data.schema_downgrade(args.version)
-            data.logger.warning("No game set, upgrading core schema")
-            data.migrate_database(
-                "CORE",
-                int(args.version) if args.version is not None else None,
-                args.action,
-            )
-
-        else:
-            data.migrate_database(
-                args.game,
-                int(args.version) if args.version is not None else None,
-                args.action,
-            )
-
-    elif args.action == "autoupgrade":
-        data.autoupgrade()
 
     elif args.action == "create-owner":
-        data.create_owner(args.email)
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.create_owner(args.email, args.access_code))
 
-    elif args.action == "migrate-card":
+    elif args.action == "migrate":
-        data.migrate_card(args.old_ac, args.new_ac, args.force)
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.migrate())
 
-    elif args.action == "cleanup":
+    elif args.action == "create-revision":
-        data.delete_hanging_users()
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.create_revision(args.message))
 
-    elif args.action == "version":
+    elif args.action == "create-autorevision":
-        data.show_versions()
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(data.create_revision_auto(args.message))
 
-    data.logger.info("Done")
+    else:
+        logging.getLogger("database").info(f"Unknown action {args.action}")

docs/INSTALL_LINUX.md

@@ -0,0 +1,107 @@
+# Installing ARTEMiS on Linux
+This guide assumes a fresh install of Debian 12 or Rasperry Pi OS. If you're using a different distrubution, your package manager commands and package names may be different then what's listed below. Please check with your repository's package manager for package names.
+
+## Install prerequisits
+### Python
+Some installs may come with python already installed. You can verify this by trying the following commands:
+- `python --version`
+- `python3 --version`
+- `python3.<minor version> --version` where `<minor version>` is a python 3 release (eg 11, 10)
+
+If your python version is at least 3.7, you can move to the next step
+
+### Libraries and other software
+ARTEMiS depends on mysql and memcached. As stated above, package names may vary by distrubution, but this is generally what you should expect to install.
+#### Rasperry Pi OS
+`sudo apt install git mariadb-server python3-pip memcached libmemcached-dev `
+
+#### Debian 12
+`sudo apt install git mariadb-server python3-pip memcached libmemcached-dev default-libmysqlclient-dev pkg-config`
+
+### Optional: Install proxy
+If you intend to use a proxy (recomended for public-facing production setups), we recomend nginx
+`sudo apt install nginx`
+
+## Database setup
+### mysql_secure_installation
+If you already have your database installed and configured, and are able to log in, skip down to the [Creating the database](#creating-the-database) section below. Otherwise, setup your newly installed database.
+
+`sudo mysql_secure_installation`
+
+Leave the root password blank, do not switch to unix socket, do reset the root password to something secure, and answer yes to the rest of the prompts. You can then log into your database with `sudo mysql`
+
+### Creating the database
+Once you're logged in, run the following commands, as root, to set up our database. Make sure you note down whatever you decide to make the password for the aime account, as you will need it to configure artemis.
+
+```sql
+CREATE USER 'aime'@'localhost' IDENTIFIED BY '<password>';
+CREATE DATABASE aime;
+GRANT Alter,Create,Delete,Drop,Index,Insert,References,Select,Update ON aime.* TO 'aime'@'localhost';
+quit
+```
+We have now set up our new user, `aime`, created a database called `aime` and given our user all the permissions it needs on every table of that database.
+
+### Configure memcached
+Under the file /etc/memcached.conf, please make sure the following parameters are set:
+
+```
+# Start with a cap of 64 megs of memory. It's reasonable, and the daemon default
+# Note that the daemon will grow to this size, but does not start out holding this much
+# memory
+
+-I 128m
+-m 1024
+```
+
+** This is mandatory to avoid memcached overload caused by Crossbeats or by massive profiles
+
+Restart memcached using:  sudo systemctl restart memcached
+
+## Getting ARTEMiS
+### Clone from gitea
+use `git clone https://gitea.tendokyu.moe/Hay1tsme/artemis.git` to pull down ARTEMiS into a folder called `artemis` created at wherever your current working directory is. `cd` into `artemis`.
+
+### Optional: Create a venv
+Python venvs are a way to install and manage packages on a per-project basis and are recomended on systems that will have multiple python scripts running on them to avoid dependancy issues. If this server will be running ARTEMiS and ONLY ARTEMiS, then it is possible to get away without creating one. If you do want to create one, you will have to install an additional package:
+
+`sudo apt install python3-venv` (like above, package name may vary depending on distro and python version)
+
+Now, simply run `python -m venv .venv` (may have to use python3 or python 3.11 instead of python) to create your virtual environment in the folder `.venv`. In order to install packages and run scripts in this environment, you have to 'activate' it by running `source .venv/bin/activate`. Your terminal should now have (venv) appended to it.
+
+### Optional: Use the develop branch
+By default, pulling down ARTEMiS from gitea will pull the `master` branch. This branch is updated less frequently, but is considered stable and ready for production use. If you'd rather have more updates, but a possibility for instability or bugs, you can switch to the develop branch by running `git checkout develop`. You can run `git checkout master` to switch back to stable.
+
+## Install python libraries
+Run `pip install -r requirements.txt` to install all of ARTEMiS' dependencies. If any installs fail, you may have missed a step in the [Install prerequisits](#install-prerequisits) section above. If you're absolutly sure you didn't, submit an issue on gitea.
+
+## Configuration
+### Copy example configs
+From the `artemis` directory, run `cp -r example_config config` to copy the example configuration files to a new folder called `config`. All of the config changes you make will be done in the `config` folder.
+
+### Optional: Generate AimeDB and Frontend JWT Secrets
+AimeDB and the frontend utalize JSON Web Tokens (JWT) for card authentication and session cookies respectivly. While generating a secret for AimeDB is optional, if you intend to run the frontend, a secret is required. You can generate a secret easily by running:
+
+`openssl rand --base64 64`
+
+With 64 being the number of bytes. You shouldn't need to go higher then 64, but you can if desired. **NOTE: When pasting secrets into the config file, make sure you remove any newlines!**
+
+### Edit `core.yaml`
+Before editing `core.yaml`, you should familiarize yourself with the name and function of each of the config options. You can find a full list in [config.md](config.md)
+
+Open `core.yaml` in the `config` folder in your prefered text editor. The only configuration option that it is absolutly mandatory to change is `aimedb`->`key`. This key must be set for the server to start, and the key must be correct, otherwise you will not be able to process aimedb requests. The correct key is floating around online, and finding it is left as an excersie to the reader. 
+
+Another option that should be changed is `database`->`password` to be the password you set when you created your database user. You did write it down somewhere, right?
+
+Since you are presumably not running the games on the same computer you're installing this server on, you're going to want to change `server`->`hostname` to be whatever hostname or IP address other PCs can reach this server by. Note that some games reject IPs and require hostnames, so setting a hostname is always recomended over an IP.
+
+### Edit game configs
+Every game has their own yaml file with settings that you may want to tweek. `InitialD Zero` and `Pokken` both have `hostname` fields in their config file that you should edit, and some games support encryption, if supplied with proper keys.
+
+### A note about IDZ
+InitialD Zero is currently the only game where it is required to specify encryption information (the AES key and at least one RSA key) for the game to start. These keys are, like the aimedb key, floating around online and will not be provided. If you don't have the keys, and don't plan on anybody connecting to your server playing InitialD Zero, it's best to set `enabled` to `False` in idz.yaml to disable the game.
+
+## Create database tables
+ARTEMiS uses alembic to manage datbase versioning. `dbutils.py` acts as a wrapper for alembic, and can execute some necessassary database functions. To create the database tables, run `python dbutils.py create`. Confirm that there are no errors, and you're good to go. If you intend to use the frontend, you may also want to run `python dbutils.py create-owner -a <your 20 digit access code here>` to create a superuser account to log in with.
+
+## Run ARTEMiS
+Once you have everything configured properly, simply run `python index.py` to start ARTEMiS. Verify that clients can connect to all services (allnet, billing, aimedb, and game servers) and setup is complete.

docs/INSTALL_WINDOWS.md

@@ -1,102 +1,77 @@
-# ARTEMiS - Windows 10/11 Guide
+# Installing ARTEMiS on Windows
-This step-by-step guide assumes that you are using a fresh install of Windows 10/11 without MySQL installed, some of the steps can be skipped if you already have an installation with MySQL 8.0 or even some of the modules already present on your environment
+This guide assumes a fresh install of Windows 10. Please be aware that due to the lack of memcached and the general woes of running a server on Windows, this is only recommended for local setups or small hosting-for-the-homies type servers.
 
-# Setup
+## Install prerequisites
-## Install Python Python 3.9 (recommended) or 3.10
+### Python
-1. Download Python 3.9 : [Link](https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe)
+- Python versions from 3.8 to 3.11 work with ARTEMiS. We recommend 3.11.
-2. Install python-3.9.13-amd64.exe
+    - https://www.python.org/ftp/python/3.11.7/python-3.11.7-amd64.exe
-    1. Select Customize installation
+- Install using whichever options best suit your environment, making sure that the Python executable is on path, such that you can open CMD, type `python --version` and see the version of Python you have installed.
-    2. Make sure that pip, tcl/tk, and the for all users are checked and hit Next
+- If you already have a working version of Python installed, skip this step.
-    3. Make sure that you enable "Create shortcuts for installed applications" and "Add Python to environment variables" and hit Install
 
-## Install MySQL 8.0
+### MariaDB
-1. Download MySQL 8.0 Server : [Link](https://dev.mysql.com/get/Downloads/MySQLInstaller/mysql-installer-community-8.0.34.0.msi)
+- It is always recommended to use MariaDB over MySQL because Oracle is a terrible company.
-2. Install mysql-installer-web-community-8.0.34.0.msi
+- While the latest release of v10 is recommended, as it is an LTS release, v11 should work fine.
-    1. Click on "Add ..." on the side
+    - https://ftp.osuosl.org/pub/mariadb//mariadb-10.11.6/winx64-packages/mariadb-10.11.6-winx64.msi
-    2. Click on the "+" next to MySQL Servers
+- REMEMBER YOUR ROOT PASSWORD SO YOU CAN LOG IN IN FUTURE STEPS.
-    3. Make sure MySQL Server 8.0.34 - X64 is under the products to be installed.
-    4. Hit Next and Next once installed
-    5. Select the configuration type "Development Computer"
-    6. Hit Next
-    7. Select "Use Legacy Authentication Method (Retain MySQL 5.x compatibility)" and hit Next
-    8. Enter a root password and then hit Next >
-    9. Leave everything under Windows Service as default and hit Next >
-    10. Click on Execute and for it to finish and hit Next> and then Finish
-3. Open MySQL 8.0 Command Line Client and login as your root user
-4. Change `<Enter Password Here>` to a new password for the user aime, type those commands to create your user and the database
 
-```sql
+### Git
-CREATE USER 'aime'@'localhost' IDENTIFIED BY '<Enter Password Here>';
+- While technically optional, it is strongly recommended to obtain ARTEMiS via git clone instead of just downloading it.
+    - https://git-scm.com/download/win
+- It is recommended to use Notepad++ as the default editor (if you have it installed), other than that, the default settings should be fine.
+
+### Optional: GUI database viewer
+- Having a GUI database editor is recommended but not required.
+- MariaDB will try to install HeidiSQL, but we recommend DBeaver.
+    - https://dbeaver.io/download/
+
+## Obtain ARTEMiS
+### Via git (recommended)
+- `git clone https://gitea.tendokyu.moe/Hay1tsme/artemis.git` via cmd in whatever folder you want to install ARTEMiS.
+    - You can switch to the develop branch for latest changes via `git checkout develop`.
+
+### Via http download
+- Download [here](https://gitea.tendokyu.moe/Hay1tsme/artemis/archive/master.zip).
+    - Develop branch can be found [here](https://gitea.tendokyu.moe/Hay1tsme/artemis/archive/develop.zip).
+- Extract the zip file somewhere.
+
+## Database setup
+- Log into your server as root, either via GUI (recommended) or CMD
+- Create the `aime` user, replace `<password>` with a password you choose. Remember it!
+```
+CREATE USER 'aime'@'localhost' IDENTIFIED BY '<password>';
 CREATE DATABASE aime;
 GRANT Alter,Create,Delete,Drop,Index,Insert,References,Select,Update ON aime.* TO 'aime'@'localhost';
-FLUSH PRIVILEGES;
-exit;
 ```
+- If you create the database via a GUI, make sure you grant all the above permissions.
 
-## Install Python modules
+## Create a venv
-1. Change your work path to the artemis-master folder using 'cd' and install the requirements:
+- Python virtual environments are a good way to manage packages and make dealing with python and pip easier.
+- `python -m pip venv venv`
+- `venv\Scripts\activate.bat` to activate the venv whenever you need to interact with ARTEMiS.
+- All the rest of the steps assume your venv is activated.
 
-```shell
+## Install pip modules
-pip install -r requirements.txt
+- `pip install -r requirements.txt`
-```	
 
-## Copy/Rename the folder `example_config` to `config`
+## Setup configuration
+- Create a new `config` folder and copy the files in `example_config` over.
+- edit `core.yaml`
+    - Put the password you created for the aime user into the `database` section.
+    - Put in the aimedb key (YOU DO NOT GENERATE THIS KEY, FIND IT SOMEWHERE).
+    - Set your hostname to be whatever hostname or IP address games can reach your server at (many games reject localhost and 127.0.0.1).
+    - Optional: generate base64-encoded secrets for aimedb and frontend.
+    - See [config.md](docs/config.md) for a full list of options.
+- edit `idz.yaml`
+    - If you don't plan on anyone using your server to play Initial D Zero, it is best to disable it to cut down on console spam on boot.
+- Edit other game yamls
+    - Add keys, set hostnames, ports, etc. Specific settings will depend on the game. See [game_specific_info](docs/game_specific_info.md).
 
-## Adjust `config/core.yaml`
+## Create Database Tables
+- `python dbutils.py create`
 
-1. Make sure to change the server `hostname` to be set to your local machine IP (ex.: 192.168.xxx.xxx) 
+## Firewall
-    - In case you want to run this only locally, set the following values:
+- If you're planning on serving games not on your PC, open at least ports 80, 8443, and 22345 in windows firewall
+    - Also set `listen_address` to either your local IP to serve on your LAN, or `0.0.0.0` for all interfaces, to accept connections from other places.
 
-```yaml
+## Start ARTEMiS
-server:
+- `python index.py`
-    listen_address: 0.0.0.0
-title: 
-    hostname: 192.168.xxx.xxx
-```
-
-1. Adjust the proper MySQL information you created earlier
-```yaml
-database:
-  host: "localhost"
-  username: "aime"
-  password: "<Enter Password Here>"
-  name: "aime"
-```
-3. Add the AimeDB key at the bottom of the file
-4. If the webui is needed, change the flag from False to True
-
-## Create the database tables for ARTEMiS
-
-```shell
-python dbutils.py create
-```
-
-## Firewall Adjustements 
-Make sure the following ports are open both on your router and local Windows firewall in case you want to use this for public use (NOT recommended):
-> Port 80 (TCP), 443 (TCP), 8443 (TCP), 22345 (TCP), 8080 (TCP), 8090 (TCP) **webui, 8444 (TCP) **mucha
-
-## Running the ARTEMiS instance
-```shell
-python index.py
-```
-
-# Troubleshooting
-
-## Game does not connect to ARTEMiS Allnet server
-1. Double-check your core.yaml, the listen_address is most likely either not binded to the proper IP or the port is not opened
-
-## Game does not connect to Title Server
-1. Verify that your core.yaml is setup properly for both the server listen_address and title hostname
-2. Boot your game and verify that an AllNet response does show and if it does, attempt to open the URI that is shown under a browser such as Edge, Chrome & Firefox.
-3. If a page is shown, the server is working properly and if it doesn't, double check your port forwarding and also that you have entered the proper local IP under the Title hostname in core.yaml.
-
-## Unhandled command under AimeDB
-1. Double check your AimeDB key under core.yaml, it is incorrect.
-
-## AttributeError: module 'collections' has no attribute 'Hashable'
-1. This means the pyYAML module is obsolete, simply rerun pip with the -U (force update) flag, as shown below.
-    - Change your work path to the artemis-master (or artemis-develop) folder using 'cd' and run the following commands:
-
-```shell
-pip install -r requirements.txt -U
-```

docs/INSTALL_WINDOWS_old.md

@@ -0,0 +1,102 @@
+# ARTEMiS - Windows 10/11 Guide
+This step-by-step guide assumes that you are using a fresh install of Windows 10/11 without MySQL installed, some of the steps can be skipped if you already have an installation with MySQL 8.0 or even some of the modules already present on your environment
+
+# Setup
+## Install Python Python 3.9 (recommended) or 3.10
+1. Download Python 3.9 : [Link](https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe)
+2. Install python-3.9.13-amd64.exe
+    1. Select Customize installation
+    2. Make sure that pip, tcl/tk, and the for all users are checked and hit Next
+    3. Make sure that you enable "Create shortcuts for installed applications" and "Add Python to environment variables" and hit Install
+
+## Install MySQL 8.0
+1. Download MySQL 8.0 Server : [Link](https://dev.mysql.com/get/Downloads/MySQLInstaller/mysql-installer-community-8.0.34.0.msi)
+2. Install mysql-installer-web-community-8.0.34.0.msi
+    1. Click on "Add ..." on the side
+    2. Click on the "+" next to MySQL Servers
+    3. Make sure MySQL Server 8.0.34 - X64 is under the products to be installed.
+    4. Hit Next and Next once installed
+    5. Select the configuration type "Development Computer"
+    6. Hit Next
+    7. Select "Use Legacy Authentication Method (Retain MySQL 5.x compatibility)" and hit Next
+    8. Enter a root password and then hit Next >
+    9. Leave everything under Windows Service as default and hit Next >
+    10. Click on Execute and for it to finish and hit Next> and then Finish
+3. Open MySQL 8.0 Command Line Client and login as your root user
+4. Change `<Enter Password Here>` to a new password for the user aime, type those commands to create your user and the database
+
+```sql
+CREATE USER 'aime'@'localhost' IDENTIFIED BY '<Enter Password Here>';
+CREATE DATABASE aime;
+GRANT Alter,Create,Delete,Drop,Index,Insert,References,Select,Update ON aime.* TO 'aime'@'localhost';
+FLUSH PRIVILEGES;
+exit;
+```
+
+## Install Python modules
+1. Change your work path to the artemis-master folder using 'cd' and install the requirements:
+
+```shell
+pip install -r requirements.txt
+```	
+
+## Copy/Rename the folder `example_config` to `config`
+
+## Adjust `config/core.yaml`
+
+1. Make sure to change the server `hostname` to be set to your local machine IP (ex.: 192.168.xxx.xxx) 
+    - In case you want to run this only locally, set the following values:
+
+```yaml
+server:
+    listen_address: 0.0.0.0
+title: 
+    hostname: 192.168.xxx.xxx
+```
+
+1. Adjust the proper MySQL information you created earlier
+```yaml
+database:
+  host: "localhost"
+  username: "aime"
+  password: "<Enter Password Here>"
+  name: "aime"
+```
+3. Add the AimeDB key at the bottom of the file
+4. If the webui is needed, change the flag from False to True
+
+## Create the database tables for ARTEMiS
+
+```shell
+python dbutils.py create
+```
+
+## Firewall Adjustements 
+Make sure the following ports are open both on your router and local Windows firewall in case you want to use this for public use (NOT recommended):
+> Port 80 (TCP), 443 (TCP), 8443 (TCP), 22345 (TCP), 8080 (TCP), 8090 (TCP) **webui, 8444 (TCP) **mucha
+
+## Running the ARTEMiS instance
+```shell
+python index.py
+```
+
+# Troubleshooting
+
+## Game does not connect to ARTEMiS Allnet server
+1. Double-check your core.yaml, the listen_address is most likely either not binded to the proper IP or the port is not opened
+
+## Game does not connect to Title Server
+1. Verify that your core.yaml is setup properly for both the server listen_address and title hostname
+2. Boot your game and verify that an AllNet response does show and if it does, attempt to open the URI that is shown under a browser such as Edge, Chrome & Firefox.
+3. If a page is shown, the server is working properly and if it doesn't, double check your port forwarding and also that you have entered the proper local IP under the Title hostname in core.yaml.
+
+## Unhandled command under AimeDB
+1. Double check your AimeDB key under core.yaml, it is incorrect.
+
+## AttributeError: module 'collections' has no attribute 'Hashable'
+1. This means the pyYAML module is obsolete, simply rerun pip with the -U (force update) flag, as shown below.
+    - Change your work path to the artemis-master (or artemis-develop) folder using 'cd' and run the following commands:
+
+```shell
+pip install -r requirements.txt -U
+```

docs/config.md

@@ -1,23 +1,24 @@
 # ARTEMiS Configuration
 ## Server
 - `listen_address`: IP Address or hostname that the server will listen for connections on. Set to 127.0.0.1 for local only, or 0.0.0.0 for all interfaces. Default `127.0.0.1`
+- `hostname`: Hostname that gets sent to clients to tell them where to connect. Games must be able to connect to your server via the hostname or IP you spcify here. Note that most games will reject `localhost` or `127.0.0.1`. Default `localhost`
+- `port`: Port that the server will listen for connections on. Default `80`
+- `ssl_key`: Location of the ssl server key for the secure title server. Ignored if you don't use SSL. Default `cert/title.key`
+- `ssl_cert`: Location of the ssl server certificate for the secure title server. Must not be a self-signed SSL. Ignored if you don't use SSL. Default `cert/title.pem`
 - `allow_user_registration`: Allows users to register in-game via the AimeDB `register` function. Disable to be able to control who can use cards on your server. Default `True`
 - `allow_unregistered_serials`: Allows games that do not have registered keychips to connect and authenticate. Disable to restrict who can connect to your server. Recomended to disable for production setups. Default `True`
 - `name`: Name for the server, used by some games in their default MOTDs. Default `ARTEMiS`
-- `is_develop`: Flags that the server is a development instance without a proxy standing in front of it. Setting to `False` tells the server not to listen for SSL, because the proxy should be handling all SSL-related things, among other things. Default `True`
+- `is_develop`: Flags that the server is a development instance, and enables some useful development features. Disable for production setups. Default `True`.
-- `threading`: Flags that `reactor.run` should be called via the `Thread` standard library. May provide a speed boost, but removes the ability to kill the server via `Ctrl + C`. Default: `False`
+- `is_using_proxy`: Flags that you'll be using some other software, such as nginx, to proxy requests, and to send `proxy_port` or `proxy_port_ssl` to games instead of `port`. Default `False`
-- `check_arcade_ip`: Checks IPs against the `arcade` table in the database, if one is defined. Default `False`
+- `proxy_port`: Which port your front-facing proxy will be listening on. Ignored if `is_using_proxy` is `False` or if set to `0`. Default `0`
-- `strict_ip_checking`: Rejects clients if there is no IP in the `arcade` table for the respective arcade
+- `proxy_port`: Which port your front-facing proxy will be listening for ssl connections on. Ignored if `is_using_proxy` is `False` or if set to `0`. Default `0`
 - `log_dir`: Directory to store logs. Server MUST have read and write permissions to this directory or you will have issues. Default `logs`
+- `check_arcade_ip`: Checks IPs against the `arcade` table in the database, if one is defined. Default `False`
+- `strict_ip_checking`: Rejects clients if there is no IP in the `arcade` table for the respective arcade. Default `False`
 ## Title
 - `loglevel`: Logging level for the title server. Default `info`
-- `hostname`: Hostname that gets sent to clients to tell them where to connect. Games must be able to connect to your server via the hostname or IP you spcify here. Note that most games will reject `localhost` or `127.0.0.1`. Default `localhost`
+- `reboot_start_time`: 24 hour JST time that clients will see as the start of maintenance period, ex `04:00`. Leave blank for no maintenance time. Default: `""`
-- `port`: Port that the title server will listen for connections on. Set to 0 to use the Allnet handler to reduce the port footprint. Default `8080`
+- `reboot_end_time`: 24 hour JST time that clients will see as the end of maintenance period, ex `05:00`. Leave blank for no maintenance time. Default: `""`
-- `port_ssl`: Port that the secure title server will listen for connections on. Set to 0 to use the Allnet handler to reduce the port footprint. Default `0`
-- `ssl_key`: Location of the ssl server key for the secure title server. Ignored if `port_ssl` is set to `0` or `is_develop` set to `False`. Default `cert/title.key`
-- `ssl_cert`: Location of the ssl server certificate for the secure title server. Must not be a self-signed SSL. Ignored if `port_ssl` is set to `0` or `is_develop` is set to `False`. Default `cert/title.pem`
-- `reboot_start_time`: 24 hour JST time that clients will see as the start of maintenance period. Leave blank for no maintenance time. Default: ""
-- `reboot_end_time`: 24 hour JST time that clients will see as the end of maintenance period. Leave blank for no maintenance time. Default: ""
 ## Database
 - `host`: Host of the database. Default `localhost`
 - `username`: Username of the account the server should connect to the database with. Default `aime`
@@ -25,24 +26,32 @@
 - `name`: Name of the database the server should expect. Default `aime`
 - `port`: Port the database server is listening on. Default `3306`
 - `protocol`: Protocol used in the connection string, e.i `mysql` would result in `mysql://...`. Default `mysql`
-- `sha2_password`: Weather or not the password in the connection string should be hashed via SHA2. Default `False`
+- `sha2_password`: Whether or not the password in the connection string should be hashed via SHA2. Default `False`
-- `loglevel`: Logging level for the database. Default `warn`
+- `loglevel`: Logging level for the database. Default `info`
-- `user_table_autoincrement_start`: What the `aime_user` table ID autoincrememnt should start with. Default `10000`
 - `memcached_host`: Host of the memcached server. Default `localhost`
 ## Frontend
-- `enable`: Weather or not the frontend should be enabled. Default `False`
+- `enable`: Whether or not the frontend servlet should run. Frontend can still be run via `python -m uvicorn core.frontend:app` even if this is set to `False`. Default `False`
-- `port`: Port the frontend should listen for connections on. Default `8090`
+- `port`: Port the frontend should listen on. Default `8080`
 - `loglevel`: Logging level for the frontend server. Default `info`
+- `secret`: Base64-encoded JWT secret for session cookies, generated by you. Default `""`
 ## Allnet
+- `standalone`: Whether allnet should launch it's own servlet on it's own port, or be part of the main servlet on the default port. Disable if you either have something proxying `naominet.jp` requests to port 80, or have port 80 set in `server` -> `port`
+- `port`: Port the allnet server should listen for connections on if it's running standalone. Games are hardcoded to ask for port `80` so only change if you have a proxy redirecting properly. Ignored if `standalone` is `False`. Default `80`
 - `loglevel`: Logging level for the allnet server. Default `info`
-- `port`: Port the allnet server should listen for connections on. Games are hardcoded to ask for port `80` so only change if you have a proxy redirecting properly. Default `80`
 - `allow_online_updates`: Allow allnet to distribute online updates via DownloadOrders. This system is currently non-functional, so leave it disabled. Default `False`
+- `update_cfg_folder`: Folder where delivery INI files will be checked for. Ignored if `allow_online_updates` is `False`. Default `""`
 ## Billing
-- `port`: Port the billing server should listen for connections on. Games are hardcoded to ask for port `8443` so only change if you have a proxy redirecting properly. Set to 0 to use the allnet handler to reduce the number of ports the server eats up. Default `8443`
+- `standalone`: Whether the billing server should launch it's own servlet on it's own port, or be part of the main servlet on the default port. Setting this to `True` requires that you have `ssl_key` and `ssl_cert` set. Default `False`
-- `ssl_key`: Location of the ssl server key for the billing server. Ignored if `port` is set to `0` or `is_develop` set to `False`. Default `cert/server.key`
+- `loglevel`: Logging level for the billing server. Default `info`
-- `ssl_cert`: Location of the ssl server certificate for the billing server. Must match the CA distributed to users or the billing server will not connect. Ignored if `port` is set to `0` or `is_develop` is set to `False`. Default `cert/server.pem`
+- `port`: Port the billing server should listen for connections on. Games are hardcoded to ask for port `8443` so only change if you have a proxy redirecting properly. Ignored if `standalone` is `False`. Default `8443`
+- `ssl_key`: Location of the ssl server key for the billing server. Ignored if `standalone` is `False`. Default `cert/server.key`
+- `ssl_cert`: Location of the ssl server certificate for the billing server. Ignored if `standalone` is `False`.  Must match the CA distributed to users or the billing server will not connect. Default `cert/server.pem`
 - `signing_key`: Location of the RSA Private key used to sign billing requests. Must match the public key distributed to users or the billing server will not connect. Default `cert/billing.key`
 ## Aimedb
+- `enable`: Whether or not aimedb should run. Default `True`
+- `listen_address`: IP Address or hostname that the aimedb server will listen for connections on. Leave this blank to use the listen address under `server`. Default `""`
 - `loglevel`: Logging level for the aimedb server. Default `info`
 - `port`: Port the aimedb server should listen for connections on. Games are hardcoded to ask for port `22345` so only change if you have a proxy redirecting properly. Default `22345`
 - `key`: Key to encrypt/decrypt aimedb requests and responses. MUST be set or the server will not start. If set incorrectly, your server will not properly handle aimedb requests. Default `""`
+- `id_secret`: Base64-encoded JWT secret for Sega Auth IDs. Leaving this blank disables this feature. Default `""`
+- `id_lifetime_seconds`: Number of secons a JWT generated should be valid for. Default `86400` (1 day)

docs/game_specific_info.md

@@ -596,7 +596,7 @@ Below is a list of VIP rewards. Currently, VIP is not implemented, and thus thes
 In order to use the importer locate your game installation folder and execute:
 
 ```shell
-python read.py --game SDEW --version <Version ID> --binfolder </path/to/game/extractedassets>
+python read.py --game SDEW --version 0 --binfolder /titles/sao/data/
 ```
 
 The importer for SAO will import all items, heroes, support skills and titles data.
@@ -623,15 +623,17 @@ python dbutils.py --game SDEW upgrade
 ### Notes
 - Defrag Match will crash at loading
 - Co-Op Online is not supported
-- Shop is not functionnal
+- Shop is displayed but cannot purchase heroes or items
 - Player title is currently static and cannot be changed in-game
 - QR Card Scanning currently only load a static hero
-
+- Ex-quests progression not supported yet
-**Network hashing in GssSite.dll must be disabled**
+- Daily Missions not implemented
+- EX TOWER 1,2 & 3 are not yet supported
+- Daily Yui coin not yet fixed
 
 ### Credits for SAO support:
 
-- Midorica - Limited Network Support
+- Midorica - Network Support
 - Dniel97 - Helping with network base
 - tungnotpunk - Source
 

example_config/chuni.yaml

@@ -25,3 +25,8 @@ version:
 
 crypto:
   encrypted_only: False
+
+matching:
+  enable: False
+  match_time_limit: 60
+  match_error_limit: 9999

example_config/core.yaml

@@ -1,26 +1,25 @@
 server:
   listen_address: "127.0.0.1"  
+  hostname: "localhost"  
+  port: 80
+  ssl_key: "cert/title.key"
+  ssl_cert: "cert/title.crt"
   allow_user_registration: True
   allow_unregistered_serials: True
   name: "ARTEMiS"
   is_develop: True
   is_using_proxy: False
-  threading: False
+  proxy_port: 0
+  proxy_port_ssl: 0
   log_dir: "logs"
   check_arcade_ip: False
   strict_ip_checking: False
 
 title:
   loglevel: "info"
-  hostname: "localhost"
-  port: 8080
-  port_ssl: 0
-  ssl_cert: "cert/title.crt"
-  ssl_key: "cert/title.key"
   reboot_start_time: "04:00"
   reboot_end_time: "05:00"
 
-
 database:
   host: "localhost"
   username: "aime"
@@ -29,30 +28,34 @@ database:
   port: 3306
   protocol: "mysql"
   sha2_password: False
-  loglevel: "warn"
+  loglevel: "info"
-  user_table_autoincrement_start: 10000
   enable_memcached: True
   memcached_host: "localhost"
 
 frontend:
   enable: False
-  port: 8090
+  port: 8080
   loglevel: "info"
+  secret: ""
 
 allnet:
-  loglevel: "info"
+  standalone: False
   port: 80
-  ip_check: False
+  loglevel: "info"
   allow_online_updates: False
   update_cfg_folder: ""
 
 billing:
+  standalone: True
+  loglevel: "info"
   port: 8443
   ssl_key: "cert/server.key"
   ssl_cert: "cert/server.pem"
   signing_key: "cert/billing.key"
 
 aimedb:
+  enable: True
+  listen_address: ""
   loglevel: "info"
   port: 22345
   key: ""
@@ -60,6 +63,4 @@ aimedb:
   id_lifetime_seconds: 86400
 
 mucha:
-  enable: False
-  hostname: "localhost"
   loglevel: "info"

example_config/cxb.yaml

@@ -1,3 +1,4 @@
 server:
   enable: True
   loglevel: "info"
+  use:https: True