Merge branch 'develop'

contributing.md

@@ -0,0 +1,8 @@
+# Contributing to ARTEMiS
+If you would like to contribute to artemis, either by adding features, games, or fixing bugs, you can do so by forking the repo and submitting a pull request [here](https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls). Please make sure, if you're submitting a PR for a game or game version, that you're following the n-0/y-1 guidelines, or it will be rejected.
+
+## Adding games
+Guide WIP
+
+## Adding game versions
+Guide WIP

core/__init__.py

@@ -4,4 +4,4 @@ from core.aimedb import AimedbFactory
 from core.title import TitleServlet
 from core.utils import Utils
 from core.mucha import MuchaServlet
-from core.frontend import FrontendServlet
+from core.frontend import FrontendServlet

core/aimedb.py

@@ -8,17 +8,18 @@ from logging.handlers import TimedRotatingFileHandler
 from core.config import CoreConfig
 from core.data import Data
 
+
 class AimedbProtocol(Protocol):
     AIMEDB_RESPONSE_CODES = {
         "felica_lookup": 0x03,
         "lookup": 0x06,
-        "log": 0x0a,
+        "log": 0x0A,
-        "campaign": 0x0c,
+        "campaign": 0x0C,
-        "touch": 0x0e,
+        "touch": 0x0E,
         "lookup2": 0x10,
         "felica_lookup2": 0x12,
         "log2": 0x14,
-        "hello": 0x65
+        "hello": 0x65,
     }
 
     request_list: Dict[int, Any] = {}
@@ -30,14 +31,14 @@ class AimedbProtocol(Protocol):
         if core_cfg.aimedb.key == "":
             self.logger.error("!!!KEY NOT SET!!!")
             exit(1)
-        
+
         self.request_list[0x01] = self.handle_felica_lookup
         self.request_list[0x04] = self.handle_lookup
         self.request_list[0x05] = self.handle_register
-        self.request_list[0x09] = self.handle_log        
+        self.request_list[0x09] = self.handle_log
-        self.request_list[0x0b] = self.handle_campaign
+        self.request_list[0x0B] = self.handle_campaign
-        self.request_list[0x0d] = self.handle_touch
+        self.request_list[0x0D] = self.handle_touch
-        self.request_list[0x0f] = self.handle_lookup2
+        self.request_list[0x0F] = self.handle_lookup2
         self.request_list[0x11] = self.handle_felica_lookup2
         self.request_list[0x13] = self.handle_log2
         self.request_list[0x64] = self.handle_hello
@@ -53,8 +54,10 @@ class AimedbProtocol(Protocol):
         self.logger.debug(f"{self.transport.getPeer().host} Connected")
 
     def connectionLost(self, reason) -> None:
-        self.logger.debug(f"{self.transport.getPeer().host} Disconnected - {reason.value}")
+        self.logger.debug(
-    
+            f"{self.transport.getPeer().host} Disconnected - {reason.value}"
+        )
+
     def dataReceived(self, data: bytes) -> None:
         cipher = AES.new(self.config.aimedb.key.encode(), AES.MODE_ECB)
 
@@ -66,7 +69,7 @@ class AimedbProtocol(Protocol):
 
         self.logger.debug(f"{self.transport.getPeer().host} wrote {decrypted.hex()}")
 
-        if not decrypted[1] == 0xa1 and not decrypted[0] == 0x3e:
+        if not decrypted[1] == 0xA1 and not decrypted[0] == 0x3E:
             self.logger.error(f"Bad magic")
             return None
 
@@ -90,30 +93,46 @@ class AimedbProtocol(Protocol):
         except ValueError as e:
             self.logger.error(f"Failed to encrypt {resp.hex()} because {e}")
             return None
-    
+
     def handle_campaign(self, data: bytes) -> bytes:
         self.logger.info(f"campaign from {self.transport.getPeer().host}")
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["campaign"], 0x0200, 0x0001)
+        ret = struct.pack(
+            "<5H",
+            0xA13E,
+            0x3087,
+            self.AIMEDB_RESPONSE_CODES["campaign"],
+            0x0200,
+            0x0001,
+        )
         return self.append_padding(ret)
-    
+
     def handle_hello(self, data: bytes) -> bytes:
         self.logger.info(f"hello from {self.transport.getPeer().host}")
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["hello"], 0x0020, 0x0001)
+        ret = struct.pack(
+            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["hello"], 0x0020, 0x0001
+        )
         return self.append_padding(ret)
 
     def handle_lookup(self, data: bytes) -> bytes:
-        luid = data[0x20: 0x2a].hex()
+        luid = data[0x20:0x2A].hex()
         user_id = self.data.card.get_user_id_from_card(access_code=luid)
 
-        if user_id is None: user_id = -1
+        if user_id is None:
+            user_id = -1
 
-        self.logger.info(f"lookup from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}")
+        self.logger.info(
+            f"lookup from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}"
+        )
 
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["lookup"], 0x0130, 0x0001)
+        ret = struct.pack(
+            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["lookup"], 0x0130, 0x0001
+        )
         ret += bytes(0x20 - len(ret))
 
-        if user_id is None: ret += struct.pack("<iH", -1, 0)
+        if user_id is None:
-        else: ret += struct.pack("<l", user_id)
+            ret += struct.pack("<iH", -1, 0)
+        else:
+            ret += struct.pack("<l", user_id)
         return self.append_padding(ret)
 
     def handle_lookup2(self, data: bytes) -> bytes:
@@ -125,66 +144,98 @@ class AimedbProtocol(Protocol):
         return bytes(ret)
 
     def handle_felica_lookup(self, data: bytes) -> bytes:
-        idm = data[0x20: 0x28].hex()
+        idm = data[0x20:0x28].hex()
-        pmm = data[0x28: 0x30].hex()
+        pmm = data[0x28:0x30].hex()
         access_code = self.data.card.to_access_code(idm)
-        self.logger.info(f"felica_lookup from {self.transport.getPeer().host}: idm {idm} pmm {pmm} -> access_code {access_code}")
+        self.logger.info(
+            f"felica_lookup from {self.transport.getPeer().host}: idm {idm} pmm {pmm} -> access_code {access_code}"
+        )
 
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["felica_lookup"], 0x0030, 0x0001)
+        ret = struct.pack(
+            "<5H",
+            0xA13E,
+            0x3087,
+            self.AIMEDB_RESPONSE_CODES["felica_lookup"],
+            0x0030,
+            0x0001,
+        )
         ret += bytes(26)
         ret += bytes.fromhex(access_code)
 
         return self.append_padding(ret)
 
     def handle_felica_lookup2(self, data: bytes) -> bytes:
-        idm = data[0x30: 0x38].hex()
+        idm = data[0x30:0x38].hex()
-        pmm = data[0x38: 0x40].hex()
+        pmm = data[0x38:0x40].hex()
         access_code = self.data.card.to_access_code(idm)
         user_id = self.data.card.get_user_id_from_card(access_code=access_code)
 
-        if user_id is None: user_id = -1
+        if user_id is None:
+            user_id = -1
 
-        self.logger.info(f"felica_lookup2 from {self.transport.getPeer().host}: idm {idm} ipm {pmm} -> access_code {access_code} user_id {user_id}")
+        self.logger.info(
+            f"felica_lookup2 from {self.transport.getPeer().host}: idm {idm} ipm {pmm} -> access_code {access_code} user_id {user_id}"
+        )
 
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["felica_lookup2"], 0x0140, 0x0001)
+        ret = struct.pack(
+            "<5H",
+            0xA13E,
+            0x3087,
+            self.AIMEDB_RESPONSE_CODES["felica_lookup2"],
+            0x0140,
+            0x0001,
+        )
         ret += bytes(22)
-        ret += struct.pack("<lq", user_id, -1) # first -1 is ext_id, 3rd is access code
+        ret += struct.pack("<lq", user_id, -1)  # first -1 is ext_id, 3rd is access code
         ret += bytes.fromhex(access_code)
         ret += struct.pack("<l", 1)
-        
+
         return self.append_padding(ret)
-    
+
     def handle_touch(self, data: bytes) -> bytes:
         self.logger.info(f"touch from {self.transport.getPeer().host}")
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["touch"], 0x0050, 0x0001)
+        ret = struct.pack(
+            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["touch"], 0x0050, 0x0001
+        )
         ret += bytes(5)
-        ret += struct.pack("<3H", 0x6f, 0, 1)
+        ret += struct.pack("<3H", 0x6F, 0, 1)
 
         return self.append_padding(ret)
 
-    def handle_register(self, data: bytes) -> bytes:        
+    def handle_register(self, data: bytes) -> bytes:
-        luid = data[0x20: 0x2a].hex()
+        luid = data[0x20:0x2A].hex()
         if self.config.server.allow_user_registration:
             user_id = self.data.user.create_user()
 
-            if user_id is None: 
+            if user_id is None:
                 user_id = -1
                 self.logger.error("Failed to register user!")
 
             else:
                 card_id = self.data.card.create_card(user_id, luid)
 
-                if card_id is None: 
+                if card_id is None:
                     user_id = -1
                     self.logger.error("Failed to register card!")
 
-            self.logger.info(f"register from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}")
+            self.logger.info(
-        
+                f"register from {self.transport.getPeer().host}: luid {luid} -> user_id {user_id}"
+            )
+
         else:
-            self.logger.info(f"register from {self.transport.getPeer().host} blocked!: luid {luid}")
+            self.logger.info(
+                f"register from {self.transport.getPeer().host} blocked!: luid {luid}"
+            )
             user_id = -1
 
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["lookup"], 0x0030, 0x0001 if user_id > -1 else 0)
+        ret = struct.pack(
+            "<5H",
+            0xA13E,
+            0x3087,
+            self.AIMEDB_RESPONSE_CODES["lookup"],
+            0x0030,
+            0x0001 if user_id > -1 else 0,
+        )
         ret += bytes(0x20 - len(ret))
         ret += struct.pack("<l", user_id)
 
@@ -193,42 +244,54 @@ class AimedbProtocol(Protocol):
     def handle_log(self, data: bytes) -> bytes:
         # TODO: Save aimedb logs
         self.logger.info(f"log from {self.transport.getPeer().host}")
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["log"], 0x0020, 0x0001)
+        ret = struct.pack(
+            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["log"], 0x0020, 0x0001
+        )
         return self.append_padding(ret)
 
     def handle_log2(self, data: bytes) -> bytes:
         self.logger.info(f"log2 from {self.transport.getPeer().host}")
-        ret = struct.pack("<5H", 0xa13e, 0x3087, self.AIMEDB_RESPONSE_CODES["log2"], 0x0040, 0x0001)
+        ret = struct.pack(
+            "<5H", 0xA13E, 0x3087, self.AIMEDB_RESPONSE_CODES["log2"], 0x0040, 0x0001
+        )
         ret += bytes(22)
         ret += struct.pack("H", 1)
 
         return self.append_padding(ret)
 
+
 class AimedbFactory(Factory):
     protocol = AimedbProtocol
+
     def __init__(self, cfg: CoreConfig) -> None:
         self.config = cfg
         log_fmt_str = "[%(asctime)s] Aimedb | %(levelname)s | %(message)s"
         log_fmt = logging.Formatter(log_fmt_str)
         self.logger = logging.getLogger("aimedb")
 
-        fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.log_dir, "aimedb"), when="d", backupCount=10)
+        fileHandler = TimedRotatingFileHandler(
+            "{0}/{1}.log".format(self.config.server.log_dir, "aimedb"),
+            when="d",
+            backupCount=10,
+        )
         fileHandler.setFormatter(log_fmt)
-        
+
         consoleHandler = logging.StreamHandler()
         consoleHandler.setFormatter(log_fmt)
 
         self.logger.addHandler(fileHandler)
         self.logger.addHandler(consoleHandler)
-        
+
         self.logger.setLevel(self.config.aimedb.loglevel)
-        coloredlogs.install(level=cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str)
+        coloredlogs.install(
-        
+            level=cfg.aimedb.loglevel, logger=self.logger, fmt=log_fmt_str
+        )
+
         if self.config.aimedb.key == "":
             self.logger.error("Please set 'key' field in your config file.")
             exit(1)
 
         self.logger.info(f"Ready on port {self.config.aimedb.port}")
-    
+
     def buildProtocol(self, addr):
         return AimedbProtocol(self.config)

core/allnet.py

@@ -12,12 +12,13 @@ from Crypto.Signature import PKCS1_v1_5
 from time import strptime
 
 from core.config import CoreConfig
-from core.data import Data
 from core.utils import Utils
+from core.data import Data
 from core.const import *
 
+
 class AllnetServlet:
-    def __init__(self, core_cfg: CoreConfig, cfg_folder: str):        
+    def __init__(self, core_cfg: CoreConfig, cfg_folder: str):
         super().__init__()
         self.config = core_cfg
         self.config_folder = cfg_folder
@@ -27,74 +28,48 @@ class AllnetServlet:
         self.logger = logging.getLogger("allnet")
         if not hasattr(self.logger, "initialized"):
             log_fmt_str = "[%(asctime)s] Allnet | %(levelname)s | %(message)s"
-            log_fmt = logging.Formatter(log_fmt_str)        
+            log_fmt = logging.Formatter(log_fmt_str)
 
-            fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.log_dir, "allnet"), when="d", backupCount=10)
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "allnet"),
+                when="d",
+                backupCount=10,
+            )
             fileHandler.setFormatter(log_fmt)
-            
+
             consoleHandler = logging.StreamHandler()
             consoleHandler.setFormatter(log_fmt)
 
             self.logger.addHandler(fileHandler)
             self.logger.addHandler(consoleHandler)
-            
+
             self.logger.setLevel(core_cfg.allnet.loglevel)
-            coloredlogs.install(level=core_cfg.allnet.loglevel, logger=self.logger, fmt=log_fmt_str)
+            coloredlogs.install(
+                level=core_cfg.allnet.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
             self.logger.initialized = True
 
         plugins = Utils.get_all_titles()
 
         if len(plugins) == 0:
             self.logger.error("No games detected!")
-        
+
         for _, mod in plugins.items():
-            for code in mod.game_codes:
+            if hasattr(mod.index, "get_allnet_info"):
-                if hasattr(mod, "use_default_title") and mod.use_default_title:
+                for code in mod.game_codes:
-                    if hasattr(mod, "include_protocol") and mod.include_protocol:
+                    enabled, uri, host = mod.index.get_allnet_info(
-                        if hasattr(mod, "title_secure") and mod.title_secure:
+                        code, self.config, self.config_folder
-                            uri = "https://"
+                    )
-                        
-                        else:
-                            uri = "http://"
 
-                    else:
+                    if enabled:
-                        uri = ""
+                        self.uri_registry[code] = (uri, host)
-                    
-                    if core_cfg.server.is_develop:
-                        uri += f"{core_cfg.title.hostname}:{core_cfg.title.port}"
-                    
-                    else:
-                        uri += f"{core_cfg.title.hostname}"
-                    
-                    uri += f"/{code}/$v"
 
-                    if hasattr(mod, "trailing_slash") and mod.trailing_slash:
+        self.logger.info(
-                        uri += "/"
+            f"Serving {len(self.uri_registry)} game codes port {core_cfg.allnet.port}"
-                
+        )
-                else:
-                    if hasattr(mod, "uri"):
-                        uri = mod.uri
-                    else:
-                        uri = ""
-
-                if hasattr(mod, "host"):
-                    host = mod.host
-                
-                elif hasattr(mod, "use_default_host") and mod.use_default_host:
-                    if core_cfg.server.is_develop:
-                        host = f"{core_cfg.title.hostname}:{core_cfg.title.port}"
-                    
-                    else:
-                        host = f"{core_cfg.title.hostname}"
-                
-                else:
-                    host = ""
-                
-                self.uri_registry[code] = (uri, host)
-        self.logger.info(f"Allnet serving {len(self.uri_registry)} games on port {core_cfg.allnet.port}")
 
     def handle_poweron(self, request: Request, _: Dict):
-        request_ip = request.getClientAddress().host
+        request_ip = Utils.get_ip_addr(request)
         try:
             req_dict = self.allnet_req_to_dict(request.content.getvalue())
             if req_dict is None:
@@ -103,15 +78,17 @@ class AllnetServlet:
             req = AllnetPowerOnRequest(req_dict[0])
             # Validate the request. Currently we only validate the fields we plan on using
 
-            if not req.game_id or not req.ver or not req.token or not req.serial or not req.ip:
+            if not req.game_id or not req.ver or not req.serial or not req.ip:
-                raise AllnetRequestException(f"Bad auth request params from {request_ip} - {vars(req)}")
+                raise AllnetRequestException(
-        
+                    f"Bad auth request params from {request_ip} - {vars(req)}"
+                )
+
         except AllnetRequestException as e:
             if e.message != "":
                 self.logger.error(e)
             return b""
-        
+
-        if req.format_ver == 3:
+        if req.format_ver == "3":
             resp = AllnetPowerOnResponse3(req.token)
         else:
             resp = AllnetPowerOnResponse2()
@@ -119,26 +96,32 @@ class AllnetServlet:
         self.logger.debug(f"Allnet request: {vars(req)}")
         if req.game_id not in self.uri_registry:
             msg = f"Unrecognised game {req.game_id} attempted allnet auth from {request_ip}."
-            self.data.base.log_event("allnet", "ALLNET_AUTH_UNKNOWN_GAME", logging.WARN, msg)
+            self.data.base.log_event(
+                "allnet", "ALLNET_AUTH_UNKNOWN_GAME", logging.WARN, msg
+            )
             self.logger.warn(msg)
 
             resp.stat = 0
             return self.dict_to_http_form_string([vars(resp)])
-        
+
         resp.uri, resp.host = self.uri_registry[req.game_id]
 
         machine = self.data.arcade.get_machine(req.serial)
         if machine is None and not self.config.server.allow_unregistered_serials:
             msg = f"Unrecognised serial {req.serial} attempted allnet auth from {request_ip}."
-            self.data.base.log_event("allnet", "ALLNET_AUTH_UNKNOWN_SERIAL", logging.WARN, msg)
+            self.data.base.log_event(
+                "allnet", "ALLNET_AUTH_UNKNOWN_SERIAL", logging.WARN, msg
+            )
             self.logger.warn(msg)
 
             resp.stat = 0
             return self.dict_to_http_form_string([vars(resp)])
-        
+
         if machine is not None:
             arcade = self.data.arcade.get_arcade(machine["arcade"])
-            country = arcade["country"] if machine["country"] is None else machine["country"]
+            country = (
+                arcade["country"] if machine["country"] is None else machine["country"]
+            )
             if country is None:
                 country = AllnetCountryCode.JAPAN.value
 
@@ -147,16 +130,30 @@ class AllnetServlet:
             resp.allnet_id = machine["id"]
             resp.name = arcade["name"] if arcade["name"] is not None else ""
             resp.nickname = arcade["nickname"] if arcade["nickname"] is not None else ""
-            resp.region0 = arcade["region_id"] if arcade["region_id"] is not None else AllnetJapanRegionId.AICHI.value
+            resp.region0 = (
-            resp.region_name0 = arcade["country"] if arcade["country"] is not None else AllnetCountryCode.JAPAN.value
+                arcade["region_id"]
-            resp.region_name1 = arcade["state"] if arcade["state"] is not None else AllnetJapanRegionId.AICHI.name
+                if arcade["region_id"] is not None
+                else AllnetJapanRegionId.AICHI.value
+            )
+            resp.region_name0 = (
+                arcade["country"]
+                if arcade["country"] is not None
+                else AllnetCountryCode.JAPAN.value
+            )
+            resp.region_name1 = (
+                arcade["state"]
+                if arcade["state"] is not None
+                else AllnetJapanRegionId.AICHI.name
+            )
             resp.region_name2 = arcade["city"] if arcade["city"] is not None else ""
-            resp.client_timezone = arcade["timezone"] if arcade["timezone"] is not None else "+0900"
+            resp.client_timezone = (
-        
+                arcade["timezone"] if arcade["timezone"] is not None else "+0900"
+            )
+
         int_ver = req.ver.replace(".", "")
         resp.uri = resp.uri.replace("$v", int_ver)
         resp.host = resp.host.replace("$v", int_ver)
-        
+
         msg = f"{req.serial} authenticated from {request_ip}: {req.game_id} v{req.ver}"
         self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
         self.logger.info(msg)
@@ -165,7 +162,7 @@ class AllnetServlet:
         return self.dict_to_http_form_string([vars(resp)]).encode("utf-8")
 
     def handle_dlorder(self, request: Request, _: Dict):
-        request_ip = request.getClientAddress().host
+        request_ip = Utils.get_ip_addr(request)
         try:
             req_dict = self.allnet_req_to_dict(request.content.getvalue())
             if req_dict is None:
@@ -175,30 +172,34 @@ class AllnetServlet:
             # Validate the request. Currently we only validate the fields we plan on using
 
             if not req.game_id or not req.ver or not req.serial:
-                raise AllnetRequestException(f"Bad download request params from {request_ip} - {vars(req)}")
+                raise AllnetRequestException(
-        
+                    f"Bad download request params from {request_ip} - {vars(req)}"
+                )
+
         except AllnetRequestException as e:
             if e.message != "":
                 self.logger.error(e)
             return b""
 
+        self.logger.info(f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}")
         resp = AllnetDownloadOrderResponse()
+        
         if not self.config.allnet.allow_online_updates:
             return self.dict_to_http_form_string([vars(resp)])
-        
+
-        else: # TODO: Actual dlorder response
+        else:  # TODO: Actual dlorder response
             return self.dict_to_http_form_string([vars(resp)])
 
     def handle_billing_request(self, request: Request, _: Dict):
         req_dict = self.billing_req_to_dict(request.content.getvalue())
-        request_ip = request.getClientAddress()
+        request_ip = Utils.get_ip_addr(request)
         if req_dict is None:
             self.logger.error(f"Failed to parse request {request.content.getvalue()}")
             return b""
-        
+
         self.logger.debug(f"request {req_dict}")
 
-        rsa = RSA.import_key(open(self.config.billing.signing_key, 'rb').read())
+        rsa = RSA.import_key(open(self.config.billing.signing_key, "rb").read())
         signer = PKCS1_v1_5.new(rsa)
         digest = SHA.new()
 
@@ -214,30 +215,34 @@ class AllnetServlet:
         machine = self.data.arcade.get_machine(kc_serial)
         if machine is None and not self.config.server.allow_unregistered_serials:
             msg = f"Unrecognised serial {kc_serial} attempted billing checkin from {request_ip} for game {kc_game}."
-            self.data.base.log_event("allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg)
+            self.data.base.log_event(
+                "allnet", "BILLING_CHECKIN_NG_SERIAL", logging.WARN, msg
+            )
             self.logger.warn(msg)
 
             resp = BillingResponse("", "", "", "")
             resp.result = "1"
             return self.dict_to_http_form_string([vars(resp)])
 
-        msg = f"Billing checkin from {request.getClientIP()}: game {kc_game} keychip {kc_serial} playcount " \
+        msg = (
+            f"Billing checkin from {request_ip}: game {kc_game} keychip {kc_serial} playcount "
             f"{kc_playcount} billing_type {kc_billigtype} nearfull {kc_nearfull} playlimit {kc_playlimit}"
+        )
         self.logger.info(msg)
-        self.data.base.log_event('billing', 'BILLING_CHECKIN_OK', logging.INFO, msg)
+        self.data.base.log_event("billing", "BILLING_CHECKIN_OK", logging.INFO, msg)
 
         while kc_playcount > kc_playlimit:
             kc_playlimit += 1024
             kc_nearfull += 1024
-        
+
         playlimit = kc_playlimit
         nearfull = kc_nearfull + (kc_billigtype * 0x00010000)
 
-        digest.update(playlimit.to_bytes(4, 'little') + kc_serial_bytes)
+        digest.update(playlimit.to_bytes(4, "little") + kc_serial_bytes)
         playlimit_sig = signer.sign(digest).hex()
 
         digest = SHA.new()
-        digest.update(nearfull.to_bytes(4, 'little') + kc_serial_bytes)
+        digest.update(nearfull.to_bytes(4, "little") + kc_serial_bytes)
         nearfull_sig = signer.sign(digest).hex()
 
         # TODO: playhistory
@@ -252,22 +257,22 @@ class AllnetServlet:
         return resp_str.encode("utf-8")
 
     def handle_naomitest(self, request: Request, _: Dict) -> bytes:
-        self.logger.info(f"Ping from {request.getClientAddress().host}")
+        self.logger.info(f"Ping from {Utils.get_ip_addr(request)}")
         return b"naomi ok"
 
     def kvp_to_dict(self, kvp: List[str]) -> List[Dict[str, Any]]:
         ret: List[Dict[str, Any]] = []
         for x in kvp:
-            items = x.split('&')
+            items = x.split("&")
             tmp = {}
 
             for item in items:
-                kvp = item.split('=')
+                kvp = item.split("=")
                 if len(kvp) == 2:
                     tmp[kvp[0]] = kvp[1]
 
             ret.append(tmp)
-        
+
         return ret
 
     def billing_req_to_dict(self, data: bytes):
@@ -277,8 +282,8 @@ class AllnetServlet:
         try:
             decomp = zlib.decompressobj(-zlib.MAX_WBITS)
             unzipped = decomp.decompress(data)
-            sections = unzipped.decode('ascii').split('\r\n')
+            sections = unzipped.decode("ascii").split("\r\n")
-            
+
             return self.kvp_to_dict(sections)
 
         except Exception as e:
@@ -288,33 +293,38 @@ class AllnetServlet:
     def allnet_req_to_dict(self, data: str) -> Optional[List[Dict[str, Any]]]:
         """
         Parses an allnet request string into a python dictionary
-        """    
+        """
         try:
             zipped = base64.b64decode(data)
             unzipped = zlib.decompress(zipped)
-            sections = unzipped.decode('utf-8').split('\r\n')
+            sections = unzipped.decode("utf-8").split("\r\n")
-            
+
             return self.kvp_to_dict(sections)
 
         except Exception as e:
             self.logger.error(f"allnet_req_to_dict: {e} while parsing {data}")
             return None
 
-    def dict_to_http_form_string(self, data:List[Dict[str, Any]], crlf: bool = False, trailing_newline: bool = True) -> Optional[str]:
+    def dict_to_http_form_string(
+        self,
+        data: List[Dict[str, Any]],
+        crlf: bool = False,
+        trailing_newline: bool = True,
+    ) -> Optional[str]:
         """
         Takes a python dictionary and parses it into an allnet response string
         """
         try:
             urlencode = ""
             for item in data:
-                for k,v in item.items():
+                for k, v in item.items():
                     urlencode += f"{k}={v}&"
 
                 if crlf:
                     urlencode = urlencode[:-1] + "\r\n"
                 else:
                     urlencode = urlencode[:-1] + "\n"
-            
+
             if not trailing_newline:
                 if crlf:
                     urlencode = urlencode[:-2]
@@ -322,31 +332,29 @@ class AllnetServlet:
                     urlencode = urlencode[:-1]
 
             return urlencode
-            
+
         except Exception as e:
             self.logger.error(f"dict_to_http_form_string: {e} while parsing {data}")
             return None
 
-class AllnetPowerOnRequest():
+
+class AllnetPowerOnRequest:
     def __init__(self, req: Dict) -> None:
         if req is None:
             raise AllnetRequestException("Request processing failed")
-        self.game_id: str = req["game_id"] if "game_id" in req else ""
+        self.game_id: str = req.get("game_id", "")
-        self.ver: str = req["ver"] if "ver" in req else ""
+        self.ver: str = req.get("ver", "")
-        self.serial: str  = req["serial"] if "serial" in req else ""
+        self.serial: str = req.get("serial", "")
-        self.ip: str  = req["ip"] if "ip" in req else ""
+        self.ip: str = req.get("ip", "")
-        self.firm_ver: str = req["firm_ver"] if "firm_ver" in req else ""
+        self.firm_ver: str = req.get("firm_ver", "")
-        self.boot_ver: str = req["boot_ver"] if "boot_ver" in req else ""
+        self.boot_ver: str = req.get("boot_ver", "")
-        self.encode: str = req["encode"] if "encode" in req else ""
+        self.encode: str = req.get("encode", "")
-        
+        self.hops = int(req.get("hops", "0"))
-        try:
+        self.format_ver = req.get("format_ver", "2")
-            self.hops = int(req["hops"]) if "hops" in req else 0
+        self.token = int(req.get("token", "0"))
-            self.format_ver = int(req["format_ver"]) if "format_ver" in req else 2
-            self.token = int(req["token"]) if "token" in req else 0
-        except ValueError as e:
-            raise AllnetRequestException(f"Failed to parse int: {e}")
 
-class AllnetPowerOnResponse3():
+
+class AllnetPowerOnResponse3:
     def __init__(self, token) -> None:
         self.stat = 1
         self.uri = ""
@@ -362,12 +370,15 @@ class AllnetPowerOnResponse3():
         self.country = "JPN"
         self.allnet_id = "123"
         self.client_timezone = "+0900"
-        self.utc_time = datetime.now(tz=pytz.timezone('UTC')).strftime("%Y-%m-%dT%H:%M:%SZ")
+        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
-        self.setting = ""
+            "%Y-%m-%dT%H:%M:%SZ"
+        )
+        self.setting = "1"
         self.res_ver = "3"
         self.token = str(token)
 
-class AllnetPowerOnResponse2():
+
+class AllnetPowerOnResponse2:
     def __init__(self) -> None:
         self.stat = 1
         self.uri = ""
@@ -391,23 +402,31 @@ class AllnetPowerOnResponse2():
         self.timezone = "+0900"
         self.res_class = "PowerOnResponseV2"
 
-class AllnetDownloadOrderRequest():
-    def __init__(self, req: Dict) -> None:
-        self.game_id = req["game_id"] if "game_id" in req else ""
-        self.ver = req["ver"] if "ver" in req else ""
-        self.serial = req["serial"] if "serial" in req else ""
-        self.encode = req["encode"] if "encode" in req else ""
 
-class AllnetDownloadOrderResponse():
+class AllnetDownloadOrderRequest:
+    def __init__(self, req: Dict) -> None:
+        self.game_id = req.get("game_id", "")
+        self.ver = req.get("ver", "")
+        self.serial = req.get("serial", "")
+        self.encode = req.get("encode", "")
+
+
+class AllnetDownloadOrderResponse:
     def __init__(self, stat: int = 1, serial: str = "", uri: str = "null") -> None:
         self.stat = stat
         self.serial = serial
         self.uri = uri
 
-class BillingResponse():
-    def __init__(self, playlimit: str = "", playlimit_sig: str = "", nearfull: str = "", nearfull_sig: str = "", 
-        playhistory: str = "000000/0:000000/0:000000/0") -> None:
 
+class BillingResponse:
+    def __init__(
+        self,
+        playlimit: str = "",
+        playlimit_sig: str = "",
+        nearfull: str = "",
+        nearfull_sig: str = "",
+        playhistory: str = "000000/0:000000/0:000000/0",
+    ) -> None:
         self.result = "0"
         self.waitime = "100"
         self.linelimit = "1"
@@ -419,10 +438,11 @@ class BillingResponse():
         self.nearfullsig = nearfull_sig
         self.fixlogincnt = "0"
         self.fixinterval = "5"
-        self.playhistory = playhistory 
+        self.playhistory = playhistory
         # playhistory -> YYYYMM/C:...
         # YYYY -> 4 digit year, MM -> 2 digit month, C -> Playcount during that period
 
+
 class AllnetRequestException(Exception):
     def __init__(self, message="") -> None:
         self.message = message

core/config.py

@@ -1,33 +1,47 @@
 import logging, os
 from typing import Any
 
+
 class ServerConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
 
     @property
     def listen_address(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'listen_address', default='127.0.0.1')
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "listen_address", default="127.0.0.1"
+        )
 
     @property
     def allow_user_registration(self) -> bool:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'allow_user_registration', default=True)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "allow_user_registration", default=True
+        )
 
     @property
     def allow_unregistered_serials(self) -> bool:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'allow_unregistered_serials', default=True)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "allow_unregistered_serials", default=True
+        )
 
     @property
     def name(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'name', default="ARTEMiS")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "name", default="ARTEMiS"
+        )
 
     @property
     def is_develop(self) -> bool:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'is_develop', default=True)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "is_develop", default=True
+        )
 
     @property
     def log_dir(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'server', 'log_dir', default='logs')
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "log_dir", default="logs"
+        )
+
 
 class TitleConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -35,15 +49,24 @@ class TitleConfig:
 
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'title', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "title", "loglevel", default="info"
+            )
+        )
 
     @property
     def hostname(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'title', 'hostname', default="localhost")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "hostname", default="localhost"
+        )
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'title', 'port', default=8080)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "title", "port", default=8080
+        )
+
 
 class DatabaseConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -51,43 +74,70 @@ class DatabaseConfig:
 
     @property
     def host(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'host', default="localhost")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "host", default="localhost"
+        )
 
     @property
     def username(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'username', default='aime')
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "username", default="aime"
+        )
 
     @property
     def password(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'password', default='aime')
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "password", default="aime"
+        )
 
     @property
     def name(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'name', default='aime')
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "name", default="aime"
+        )
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'port', default=3306)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "port", default=3306
+        )
 
     @property
     def protocol(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'type', default="mysql")
+        return CoreConfig.get_config_field(
-    
+            self.__config, "core", "database", "type", default="mysql"
+        )
+
     @property
     def sha2_password(self) -> bool:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'sha2_password', default=False)
+        return CoreConfig.get_config_field(
-    
+            self.__config, "core", "database", "sha2_password", default=False
+        )
+
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'database', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "database", "loglevel", default="info"
+            )
+        )
 
     @property
     def user_table_autoincrement_start(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'user_table_autoincrement_start', default=10000)
+        return CoreConfig.get_config_field(
-    
+            self.__config,
+            "core",
+            "database",
+            "user_table_autoincrement_start",
+            default=10000,
+        )
+
     @property
     def memcached_host(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'database', 'memcached_host', default="localhost")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "database", "memcached_host", default="localhost"
+        )
+
 
 class FrontendConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -95,15 +145,24 @@ class FrontendConfig:
 
     @property
     def enable(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'frontend', 'enable', default=False)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "frontend", "enable", default=False
+        )
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'frontend', 'port', default=8090)
+        return CoreConfig.get_config_field(
-    
+            self.__config, "core", "frontend", "port", default=8090
+        )
+
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'frontend', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "frontend", "loglevel", default="info"
+            )
+        )
+
 
 class AllnetConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -111,15 +170,24 @@ class AllnetConfig:
 
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'allnet', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "allnet", "loglevel", default="info"
+            )
+        )
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'allnet', 'port', default=80)
+        return CoreConfig.get_config_field(
-    
+            self.__config, "core", "allnet", "port", default=80
+        )
+
     @property
     def allow_online_updates(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'allnet', 'allow_online_updates', default=False)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "allow_online_updates", default=False
+        )
+
 
 class BillingConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -127,35 +195,53 @@ class BillingConfig:
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'billing', 'port', default=8443)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "port", default=8443
+        )
 
     @property
     def ssl_key(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'billing', 'ssl_key', default="cert/server.key")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "ssl_key", default="cert/server.key"
+        )
 
     @property
     def ssl_cert(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'billing', 'ssl_cert', default="cert/server.pem")
+        return CoreConfig.get_config_field(
-    
+            self.__config, "core", "billing", "ssl_cert", default="cert/server.pem"
+        )
+
     @property
     def signing_key(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'billing', 'signing_key', default="cert/billing.key")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "billing", "signing_key", default="cert/billing.key"
+        )
+
 
 class AimedbConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
         self.__config = parent_config
-    
+
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'aimedb', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "core", "aimedb", "loglevel", default="info"
+            )
+        )
 
     @property
     def port(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'aimedb', 'port', default=22345)
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "port", default=22345
+        )
 
     @property
     def key(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'aimedb', 'key', default="")
+        return CoreConfig.get_config_field(
+            self.__config, "core", "aimedb", "key", default=""
+        )
+
 
 class MuchaConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:
@@ -163,27 +249,24 @@ class MuchaConfig:
 
     @property
     def enable(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'enable', default=False)
+        return CoreConfig.get_config_field(
-    
+            self.__config, "core", "mucha", "enable", default=False
-    @property
+        )
-    def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'loglevel', default="info"))
-    
-    @property
-    def hostname(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'hostname', default="localhost")
 
     @property
-    def port(self) -> int:
+    def loglevel(self) -> int:
-        return CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'port', default=8444)
+        return CoreConfig.str_to_loglevel(
-    
+            CoreConfig.get_config_field(
+                self.__config, "core", "mucha", "loglevel", default="info"
+            )
+        )
+
     @property
-    def ssl_cert(self) -> str:
+    def hostname(self) -> str:
-        return CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'ssl_cert', default="cert/server.pem")
+        return CoreConfig.get_config_field(
-    
+            self.__config, "core", "mucha", "hostname", default="localhost"
-    @property
+        )
-    def signing_key(self) -> str:
+
-        return CoreConfig.get_config_field(self.__config, 'core', 'mucha', 'signing_key', default="cert/billing.key")
 
 class CoreConfig(dict):
     def __init__(self) -> None:
@@ -200,20 +283,22 @@ class CoreConfig(dict):
     def str_to_loglevel(cls, level_str: str):
         if level_str.lower() == "error":
             return logging.ERROR
-        elif level_str.lower().startswith("warn"): # Fits warn or warning
+        elif level_str.lower().startswith("warn"):  # Fits warn or warning
             return logging.WARN
         elif level_str.lower() == "debug":
             return logging.DEBUG
         else:
-             return logging.INFO
+            return logging.INFO
 
     @classmethod
-    def get_config_field(cls, __config: dict, module, *path: str, default: Any = "") -> Any:
+    def get_config_field(
-        envKey = f'CFG_{module}_'
+        cls, __config: dict, module, *path: str, default: Any = ""
+    ) -> Any:
+        envKey = f"CFG_{module}_"
         for arg in path:
-            envKey += arg + '_'
+            envKey += arg + "_"
-        
+
-        if envKey.endswith('_'):
+        if envKey.endswith("_"):
             envKey = envKey[:-1]
 
         if envKey in os.environ:

core/const.py

@@ -1,6 +1,7 @@
 from enum import Enum
 
-class MainboardPlatformCodes():
+
+class MainboardPlatformCodes:
     RINGEDGE = "AALE"
     RINGWIDE = "AAML"
     NU = "AAVE"
@@ -8,7 +9,8 @@ class MainboardPlatformCodes():
     ALLS_UX = "ACAE"
     ALLS_HX = "ACAX"
 
-class MainboardRevisions():
+
+class MainboardRevisions:
     RINGEDGE = 1
     RINGEDGE2 = 2
 
@@ -26,12 +28,14 @@ class MainboardRevisions():
     ALLS_UX2 = 2
     ALLS_HX2 = 12
 
-class KeychipPlatformsCodes():
+
+class KeychipPlatformsCodes:
     RING = "A72E"
     NU = ("A60E", "A60E", "A60E")
     NUSX = ("A61X", "A69X")
     ALLS = "A63E"
 
+
 class AllnetCountryCode(Enum):
     JAPAN = "JPN"
     UNITED_STATES = "USA"
@@ -41,6 +45,7 @@ class AllnetCountryCode(Enum):
     TAIWAN = "TWN"
     CHINA = "CHN"
 
+
 class AllnetJapanRegionId(Enum):
     NONE = 0
     AICHI = 1

core/data/__init__.py

@@ -1,2 +1,2 @@
 from core.data.database import Data
-from core.data.cache import cached
+from core.data.cache import cached

core/data/cache.py

@@ -1,4 +1,3 @@
-
 from typing import Any, Callable
 from functools import wraps
 import hashlib
@@ -6,15 +5,17 @@ import pickle
 import logging
 from core.config import CoreConfig
 
-cfg:CoreConfig = None # type: ignore
+cfg: CoreConfig = None  # type: ignore
 # Make memcache optional
 try:
     import pylibmc  # type: ignore
+
     has_mc = True
 except ModuleNotFoundError:
     has_mc = False
 
-def cached(lifetime: int=10, extra_key: Any=None) -> Callable:
+
+def cached(lifetime: int = 10, extra_key: Any = None) -> Callable:
     def _cached(func: Callable) -> Callable:
         if has_mc:
             hostname = "127.0.0.1"
@@ -22,11 +23,10 @@ def cached(lifetime: int=10, extra_key: Any=None) -> Callable:
                 hostname = cfg.database.memcached_host
             memcache = pylibmc.Client([hostname], binary=True)
             memcache.behaviors = {"tcp_nodelay": True, "ketama": True}
-            
+
             @wraps(func)
             def wrapper(*args: Any, **kwargs: Any) -> Any:
                 if lifetime is not None:
-
                     # Hash function args
                     items = kwargs.items()
                     hashable_args = (args[1:], sorted(list(items)))
@@ -41,7 +41,7 @@ def cached(lifetime: int=10, extra_key: Any=None) -> Callable:
                     except pylibmc.Error as e:
                         logging.getLogger("database").error(f"Memcache failed: {e}")
                         result = None
-                        
+
                     if result is not None:
                         logging.getLogger("database").debug(f"Cache hit: {result}")
                         return result
@@ -55,7 +55,9 @@ def cached(lifetime: int=10, extra_key: Any=None) -> Callable:
                     memcache.set(cache_key, result, lifetime)
 
                 return result
+
         else:
+
             @wraps(func)
             def wrapper(*args: Any, **kwargs: Any) -> Any:
                 return func(*args, **kwargs)

core/data/database.py

@@ -1,18 +1,19 @@
 import logging, coloredlogs
-from typing import Any, Dict, List
+from typing import Optional
 from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
 from logging.handlers import TimedRotatingFileHandler
-from datetime import datetime
+import importlib, os
-import importlib, os, json
+import secrets, string
-
+import bcrypt
 from hashlib import sha256
 
 from core.config import CoreConfig
 from core.data.schema import *
 from core.utils import Utils
 
+
 class Data:
     def __init__(self, cfg: CoreConfig) -> None:
         self.config = cfg
@@ -22,7 +23,7 @@ class Data:
             self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{passwd.hex()}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
         else:
             self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
-        
+
         self.__engine = create_engine(self.__url, pool_recycle=3600)
         session = sessionmaker(bind=self.__engine, autoflush=True, autocommit=True)
         self.session = scoped_session(session)
@@ -31,18 +32,22 @@ class Data:
         self.arcade = ArcadeData(self.config, self.session)
         self.card = CardData(self.config, self.session)
         self.base = BaseData(self.config, self.session)
-        self.schema_ver_latest = 2
+        self.schema_ver_latest = 4
 
         log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
         log_fmt = logging.Formatter(log_fmt_str)
         self.logger = logging.getLogger("database")
 
         # Prevent the logger from adding handlers multiple times
-        if not getattr(self.logger, 'handler_set', None):
+        if not getattr(self.logger, "handler_set", None):
-            fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.log_dir, "db"), encoding="utf-8",
+            fileHandler = TimedRotatingFileHandler(
-                when="d", backupCount=10)
+                "{0}/{1}.log".format(self.config.server.log_dir, "db"),
+                encoding="utf-8",
+                when="d",
+                backupCount=10,
+            )
             fileHandler.setFormatter(log_fmt)
-            
+
             consoleHandler = logging.StreamHandler()
             consoleHandler.setFormatter(log_fmt)
 
@@ -50,8 +55,10 @@ class Data:
             self.logger.addHandler(consoleHandler)
 
             self.logger.setLevel(self.config.database.loglevel)
-            coloredlogs.install(cfg.database.loglevel, logger=self.logger, fmt=log_fmt_str)
+            coloredlogs.install(
-            self.logger.handler_set = True # type: ignore
+                cfg.database.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.handler_set = True  # type: ignore
 
     def create_database(self):
         self.logger.info("Creating databases...")
@@ -60,24 +67,33 @@ class Data:
         except SQLAlchemyError as e:
             self.logger.error(f"Failed to create databases! {e}")
             return
-        
+
         games = Utils.get_all_titles()
         for game_dir, game_mod in games.items():
             try:
-                title_db = game_mod.database(self.config)
+                if hasattr(game_mod, "database") and hasattr(game_mod, "current_schema_version"):
+                    game_mod.database(self.config)
                 metadata.create_all(self.__engine.connect())
 
-                self.base.set_schema_ver(game_mod.current_schema_version, game_mod.game_codes[0])
+                self.base.set_schema_ver(
+                    game_mod.current_schema_version, game_mod.game_codes[0]
+                )
 
             except Exception as e:
-                self.logger.warning(f"Could not load database schema from {game_dir} - {e}")
+                self.logger.warning(
-        
+                    f"Could not load database schema from {game_dir} - {e}"
+                )
+
         self.logger.info(f"Setting base_schema_ver to {self.schema_ver_latest}")
         self.base.set_schema_ver(self.schema_ver_latest)
 
-        self.logger.info(f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}")
+        self.logger.info(
-        self.user.reset_autoincrement(self.config.database.user_table_autoincrement_start)
+            f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
-    
+        )
+        self.user.reset_autoincrement(
+            self.config.database.user_table_autoincrement_start
+        )
+
     def recreate_database(self):
         self.logger.info("Dropping all databases...")
         self.base.execute("SET FOREIGN_KEY_CHECKS=0")
@@ -86,55 +102,178 @@ class Data:
         except SQLAlchemyError as e:
             self.logger.error(f"Failed to drop databases! {e}")
             return
-        
+
         for root, dirs, files in os.walk("./titles"):
-            for dir in dirs: 
+            for dir in dirs:
                 if not dir.startswith("__"):
                     try:
                         mod = importlib.import_module(f"titles.{dir}")
-                        
+
                         try:
-                            title_db = mod.database(self.config)
+                            if hasattr(mod, "database"):
+                                mod.database(self.config)
                             metadata.drop_all(self.__engine.connect())
 
                         except Exception as e:
-                            self.logger.warning(f"Could not load database schema from {dir} - {e}")
+                            self.logger.warning(
+                                f"Could not load database schema from {dir} - {e}"
+                            )
 
                     except ImportError as e:
-                        self.logger.warning(f"Failed to load database schema dir {dir} - {e}")
+                        self.logger.warning(
+                            f"Failed to load database schema dir {dir} - {e}"
+                        )
             break
-        
+
         self.base.execute("SET FOREIGN_KEY_CHECKS=1")
 
         self.create_database()
-    
+
     def migrate_database(self, game: str, version: int, action: str) -> None:
         old_ver = self.base.get_schema_ver(game)
         sql = ""
-        
+
         if old_ver is None:
-            self.logger.error(f"Schema for game {game} does not exist, did you run the creation script?")
+            self.logger.error(
-            return
+                f"Schema for game {game} does not exist, did you run the creation script?"
-        
+            )
-        if old_ver == version:
-            self.logger.info(f"Schema for game {game} is already version {old_ver}, nothing to do")
-            return
-        
-        if not os.path.exists(f"core/data/schema/versions/{game.upper()}_{version}_{action}.sql"):            
-            self.logger.error(f"Could not find {action} script {game.upper()}_{version}_{action}.sql in core/data/schema/versions folder")
             return
 
-        with open(f"core/data/schema/versions/{game.upper()}_{version}_{action}.sql", "r", encoding="utf-8") as f:
+        if old_ver == version:
-            sql = f.read()
+            self.logger.info(
-        
+                f"Schema for game {game} is already version {old_ver}, nothing to do"
-        result = self.base.execute(sql)
+            )
-        if result is None:
+            return
-            self.logger.error("Error execuing sql script!")
+
-            return None
+        if action == "upgrade":
+            for x in range(old_ver, version):
+                if not os.path.exists(
+                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql"
+                ):
+                    self.logger.error(
+                        f"Could not find {action} script {game.upper()}_{x + 1}_{action}.sql in core/data/schema/versions folder"
+                    )
+                    return
+
+                with open(
+                    f"core/data/schema/versions/{game.upper()}_{x + 1}_{action}.sql",
+                    "r",
+                    encoding="utf-8",
+                ) as f:
+                    sql = f.read()
+
+                result = self.base.execute(sql)
+                if result is None:
+                    self.logger.error("Error execuing sql script!")
+                    return None
         
+        else:
+            for x in range(old_ver, version, -1):
+                if not os.path.exists(
+                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql"
+                ):
+                    self.logger.error(
+                        f"Could not find {action} script {game.upper()}_{x - 1}_{action}.sql in core/data/schema/versions folder"
+                    )
+                    return
+
+                with open(
+                    f"core/data/schema/versions/{game.upper()}_{x - 1}_{action}.sql",
+                    "r",
+                    encoding="utf-8",
+                ) as f:
+                    sql = f.read()
+
+                result = self.base.execute(sql)
+                if result is None:
+                    self.logger.error("Error execuing sql script!")
+                    return None
+
         result = self.base.set_schema_ver(version, game)
         if result is None:
             self.logger.error("Error setting version in schema_version table!")
             return None
-        
+
         self.logger.info(f"Successfully migrated {game} to schema version {version}")
+
+    def create_owner(self, email: Optional[str] = None) -> None:
+        pw = "".join(
+            secrets.choice(string.ascii_letters + string.digits) for i in range(20)
+        )
+        hash = bcrypt.hashpw(pw.encode(), bcrypt.gensalt())
+
+        user_id = self.user.create_user(email=email, permission=255, password=hash)
+        if user_id is None:
+            self.logger.error(f"Failed to create owner with email {email}")
+            return
+
+        card_id = self.card.create_card(user_id, "00000000000000000000")
+        if card_id is None:
+            self.logger.error(f"Failed to create card for owner with id {user_id}")
+            return
+
+        self.logger.warn(
+            f"Successfully created owner with email {email}, access code 00000000000000000000, and password {pw} Make sure to change this password and assign a real card ASAP!"
+        )
+
+    def migrate_card(self, old_ac: str, new_ac: str, should_force: bool) -> None:
+        if old_ac == new_ac:
+            self.logger.error("Both access codes are the same!")
+            return
+
+        new_card = self.card.get_card_by_access_code(new_ac)
+        if new_card is None:
+            self.card.update_access_code(old_ac, new_ac)
+            return
+
+        if not should_force:
+            self.logger.warn(
+                f"Card already exists for access code {new_ac} (id {new_card['id']}). If you wish to continue, rerun with the '--force' flag."
+                f" All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
+            )
+            return
+
+        self.logger.info(
+            f"All exiting data on the target card {new_ac} will be perminently erased and replaced with data from card {old_ac}."
+        )
+        self.card.delete_card(new_card["id"])
+        self.card.update_access_code(old_ac, new_ac)
+
+        hanging_user = self.user.get_user(new_card["user"])
+        if hanging_user["password"] is None:
+            self.logger.info(f"Delete hanging user {hanging_user['id']}")
+            self.user.delete_user(hanging_user["id"])
+
+    def delete_hanging_users(self) -> None:
+        """
+        Finds and deletes users that have not registered for the webui that have no cards assocated with them.
+        """
+        unreg_users = self.user.get_unregistered_users()
+        if unreg_users is None:
+            self.logger.error("Error occoured finding unregistered users")
+
+        for user in unreg_users:
+            cards = self.card.get_user_cards(user["id"])
+            if cards is None:
+                self.logger.error(f"Error getting cards for user {user['id']}")
+                continue
+
+            if not cards:
+                self.logger.info(f"Delete hanging user {user['id']}")
+                self.user.delete_user(user["id"])
+
+    def autoupgrade(self) -> None:
+        all_games = self.base.get_all_schema_vers()
+        if all_games is None:
+            self.logger.warn("Failed to get schema versions")
+        
+        for x in all_games:
+            game = x["game"].upper()
+            update_ver = 1
+            for y in range(2, 100):
+                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
+                    update_ver = y
+                else:
+                    break
+        
+            self.migrate_database(game, update_ver, "upgrade")

core/data/schema/__init__.py

@@ -3,4 +3,4 @@ from core.data.schema.card import CardData
 from core.data.schema.base import BaseData, metadata
 from core.data.schema.arcade import ArcadeData
 
-__all__ = ["UserData", "CardData", "BaseData", "metadata", "ArcadeData"]
+__all__ = ["UserData", "CardData", "BaseData", "metadata", "ArcadeData"]

core/data/schema/arcade.py

@@ -14,131 +14,186 @@ arcade = Table(
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
     Column("name", String(255)),
-    Column("nickname", String(255)),    
+    Column("nickname", String(255)),
     Column("country", String(3)),
     Column("country_id", Integer),
     Column("state", String(255)),
     Column("city", String(255)),
     Column("region_id", Integer),
     Column("timezone", String(255)),
-    mysql_charset='utf8mb4'
+    mysql_charset="utf8mb4",
 )
 
 machine = Table(
     "machine",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
-    Column("arcade", ForeignKey("arcade.id", ondelete="cascade", onupdate="cascade"), nullable=False),    
+    Column(
+        "arcade",
+        ForeignKey("arcade.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
     Column("serial", String(15), nullable=False),
     Column("board", String(15)),
     Column("game", String(4)),
-    Column("country", String(3)), # overwrites if not null
+    Column("country", String(3)),  # overwrites if not null
     Column("timezone", String(255)),
     Column("ota_enable", Boolean),
     Column("is_cab", Boolean),
-    mysql_charset='utf8mb4'
+    mysql_charset="utf8mb4",
 )
 
 arcade_owner = Table(
-    'arcade_owner',
+    "arcade_owner",
     metadata,
-    Column('user', Integer, ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column(
-    Column('arcade', Integer, ForeignKey("arcade.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+        "user",
-    Column('permissions', Integer, nullable=False),
+        Integer,
-    PrimaryKeyConstraint('user', 'arcade', name='arcade_owner_pk'),
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
-    mysql_charset='utf8mb4'
+        nullable=False,
+    ),
+    Column(
+        "arcade",
+        Integer,
+        ForeignKey("arcade.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("permissions", Integer, nullable=False),
+    PrimaryKeyConstraint("user", "arcade", name="arcade_owner_pk"),
+    mysql_charset="utf8mb4",
 )
 
+
 class ArcadeData(BaseData):
     def get_machine(self, serial: str = None, id: int = None) -> Optional[Dict]:
         if serial is not None:
             serial = serial.replace("-", "")
             if len(serial) == 11:
                 sql = machine.select(machine.c.serial.like(f"{serial}%"))
-            
+
             elif len(serial) == 15:
                 sql = machine.select(machine.c.serial == serial)
-            
+
             else:
                 self.logger.error(f"{__name__ }: Malformed serial {serial}")
                 return None
-        
+
         elif id is not None:
             sql = machine.select(machine.c.id == id)
-        
+
-        else: 
+        else:
             self.logger.error(f"{__name__ }: Need either serial or ID to look up!")
             return None
-        
+
         result = self.execute(sql)
-        if result is None: return None
+        if result is None:
+            return None
         return result.fetchone()
-    
+
-    def put_machine(self, arcade_id: int, serial: str = "", board: str = None, game: str = None, is_cab: bool = False) -> Optional[int]:
+    def put_machine(
+        self,
+        arcade_id: int,
+        serial: str = "",
+        board: str = None,
+        game: str = None,
+        is_cab: bool = False,
+    ) -> Optional[int]:
         if arcade_id:
             self.logger.error(f"{__name__ }: Need arcade id!")
             return None
 
-        sql = machine.insert().values(arcade = arcade_id, keychip = serial, board = board, game = game, is_cab = is_cab)
+        sql = machine.insert().values(
-        
+            arcade=arcade_id, keychip=serial, board=board, game=game, is_cab=is_cab
-        result = self.execute(sql)
-        if result is None: return None
-        return result.lastrowid
-    
-    def set_machine_serial(self, machine_id: int, serial: str) -> None:
-        result = self.execute(machine.update(machine.c.id == machine_id).values(keychip = serial))
-        if result is None:
-            self.logger.error(f"Failed to update serial for machine {machine_id} -> {serial}")
-        return result.lastrowid
-    
-    def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
-        result = self.execute(machine.update(machine.c.id == machine_id).values(board = boardid))
-        if result is None:
-            self.logger.error(f"Failed to update board id for machine {machine_id} -> {boardid}")
-    
-    def get_arcade(self, id: int) -> Optional[Dict]:
-        sql = arcade.select(arcade.c.id == id)
-        result = self.execute(sql)
-        if result is None: return None
-        return result.fetchone()
-    
-    def put_arcade(self, name: str, nickname: str = None, country: str = "JPN", country_id: int = 1, 
-        state: str = "", city: str = "", regional_id: int = 1) -> Optional[int]:
-        if nickname is None: nickname = name
-
-        sql = arcade.insert().values(name = name, nickname = nickname, country = country, country_id = country_id,
-        state = state, city = city, regional_id = regional_id)
-
-        result = self.execute(sql)
-        if result is None: return None
-        return result.lastrowid
-
-    def get_arcade_owners(self, arcade_id: int) -> Optional[Dict]:
-        sql = select(arcade_owner).where(arcade_owner.c.arcade==arcade_id)
-
-        result = self.execute(sql)
-        if result is None: return None
-        return result.fetchall()
-
-    def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
-        sql = insert(arcade_owner).values(
-            arcade=arcade_id,
-            user=user_id
         )
 
         result = self.execute(sql)
-        if result is None: return None
+        if result is None:
+            return None
         return result.lastrowid
 
-    def format_serial(self, platform_code: str, platform_rev: int, serial_num: int, append: int = 4152) -> str:
+    def set_machine_serial(self, machine_id: int, serial: str) -> None:
-        return f"{platform_code}{platform_rev:02d}A{serial_num:04d}{append:04d}" # 0x41 = A, 0x52 = R
+        result = self.execute(
-    
+            machine.update(machine.c.id == machine_id).values(keychip=serial)
+        )
+        if result is None:
+            self.logger.error(
+                f"Failed to update serial for machine {machine_id} -> {serial}"
+            )
+        return result.lastrowid
+
+    def set_machine_boardid(self, machine_id: int, boardid: str) -> None:
+        result = self.execute(
+            machine.update(machine.c.id == machine_id).values(board=boardid)
+        )
+        if result is None:
+            self.logger.error(
+                f"Failed to update board id for machine {machine_id} -> {boardid}"
+            )
+
+    def get_arcade(self, id: int) -> Optional[Dict]:
+        sql = arcade.select(arcade.c.id == id)
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_arcade(
+        self,
+        name: str,
+        nickname: str = None,
+        country: str = "JPN",
+        country_id: int = 1,
+        state: str = "",
+        city: str = "",
+        regional_id: int = 1,
+    ) -> Optional[int]:
+        if nickname is None:
+            nickname = name
+
+        sql = arcade.insert().values(
+            name=name,
+            nickname=nickname,
+            country=country,
+            country_id=country_id,
+            state=state,
+            city=city,
+            regional_id=regional_id,
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def get_arcade_owners(self, arcade_id: int) -> Optional[Dict]:
+        sql = select(arcade_owner).where(arcade_owner.c.arcade == arcade_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def add_arcade_owner(self, arcade_id: int, user_id: int) -> None:
+        sql = insert(arcade_owner).values(arcade=arcade_id, user=user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def format_serial(
+        self, platform_code: str, platform_rev: int, serial_num: int, append: int = 4152
+    ) -> str:
+        return f"{platform_code}{platform_rev:02d}A{serial_num:04d}{append:04d}"  # 0x41 = A, 0x52 = R
+
     def validate_keychip_format(self, serial: str) -> bool:
         serial = serial.replace("-", "")
         if len(serial) != 11 or len(serial) != 15:
-            self.logger.error(f"Serial validate failed: Incorrect length for {serial} (len {len(serial)})")
+            self.logger.error(
+                f"Serial validate failed: Incorrect length for {serial} (len {len(serial)})"
+            )
             return False
-        
+
         platform_code = serial[:4]
         platform_rev = serial[4:6]
         const_a = serial[6]
@@ -150,11 +205,15 @@ class ArcadeData(BaseData):
             return False
 
         if len(append) != 0 or len(append) != 4:
-            self.logger.error(f"Serial validate failed: {serial} had malformed append {append}")
+            self.logger.error(
+                f"Serial validate failed: {serial} had malformed append {append}"
+            )
             return False
 
         if len(num) != 4:
-            self.logger.error(f"Serial validate failed: {serial} had malformed number {num}")
+            self.logger.error(
+                f"Serial validate failed: {serial} had malformed number {num}"
+            )
             return False
-        
+
         return True

core/data/schema/base.py

@@ -2,6 +2,7 @@ import json
 import logging
 from random import randrange
 from typing import Any, Optional, Dict, List
+from sqlalchemy.engine import Row
 from sqlalchemy.engine.cursor import CursorResult
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.sql import text, func, select
@@ -19,7 +20,7 @@ schema_ver = Table(
     metadata,
     Column("game", String(4), primary_key=True, nullable=False),
     Column("version", Integer, nullable=False, server_default="1"),
-    mysql_charset='utf8mb4'
+    mysql_charset="utf8mb4",
 )
 
 event_log = Table(
@@ -29,18 +30,20 @@ event_log = Table(
     Column("system", String(255), nullable=False),
     Column("type", String(255), nullable=False),
     Column("severity", Integer, nullable=False),
+    Column("message", String(1000), nullable=False),
     Column("details", JSON, nullable=False),
     Column("when_logged", TIMESTAMP, nullable=False, server_default=func.now()),
-    mysql_charset='utf8mb4'
+    mysql_charset="utf8mb4",
 )
 
-class BaseData():
+
+class BaseData:
     def __init__(self, cfg: CoreConfig, conn: Connection) -> None:
         self.config = cfg
         self.conn = conn
         self.logger = logging.getLogger("database")
-    
+
-    def execute(self, sql: str, opts: Dict[str, Any]={}) -> Optional[CursorResult]:
+    def execute(self, sql: str, opts: Dict[str, Any] = {}) -> Optional[CursorResult]:
         res = None
 
         try:
@@ -50,7 +53,7 @@ class BaseData():
         except SQLAlchemyError as e:
             self.logger.error(f"SQLAlchemy error {e}")
             return None
-        
+
         except UnicodeEncodeError as e:
             self.logger.error(f"UnicodeEncodeError error {e}")
             return None
@@ -62,7 +65,7 @@ class BaseData():
             except SQLAlchemyError as e:
                 self.logger.error(f"SQLAlchemy error {e}")
                 return None
-            
+
             except UnicodeEncodeError as e:
                 self.logger.error(f"UnicodeEncodeError error {e}")
                 return None
@@ -72,53 +75,79 @@ class BaseData():
                 raise
 
         return res
-    
+
     def generate_id(self) -> int:
         """
         Generate a random 5-7 digit id
         """
         return randrange(10000, 9999999)
     
+    def get_all_schema_vers(self) -> Optional[List[Row]]:
+        sql = select(schema_ver)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def get_schema_ver(self, game: str) -> Optional[int]:
         sql = select(schema_ver).where(schema_ver.c.game == game)
 
         result = self.execute(sql)
         if result is None:
             return None
-        return result.fetchone()["version"]
+
-    
+        row = result.fetchone()
+        if row is None:
+            return None
+
+        return row["version"]
+
     def set_schema_ver(self, ver: int, game: str = "CORE") -> Optional[int]:
-        sql = insert(schema_ver).values(game = game, version = ver)
+        sql = insert(schema_ver).values(game=game, version=ver)
-        conflict = sql.on_duplicate_key_update(version = ver)
+        conflict = sql.on_duplicate_key_update(version=ver)
-        
+
         result = self.execute(conflict)
         if result is None:
-            self.logger.error(f"Failed to update schema version for game {game} (v{ver})")
+            self.logger.error(
+                f"Failed to update schema version for game {game} (v{ver})"
+            )
             return None
         return result.lastrowid
 
-    def log_event(self, system: str, type: str, severity: int, details: Dict) -> Optional[int]:
+    def log_event(
-        sql = event_log.insert().values(system = system, type = type, severity = severity, details = json.dumps(details))
+        self, system: str, type: str, severity: int, message: str, details: Dict = {}
+    ) -> Optional[int]:
+        sql = event_log.insert().values(
+            system=system,
+            type=type,
+            severity=severity,
+            message=message,
+            details=json.dumps(details),
+        )
         result = self.execute(sql)
 
         if result is None:
-            self.logger.error(f"{__name__}: Failed to insert event into event log! system = {system}, type = {type}, severity = {severity}, details = {details}")
+            self.logger.error(
+                f"{__name__}: Failed to insert event into event log! system = {system}, type = {type}, severity = {severity}, message = {message}"
+            )
             return None
 
         return result.lastrowid
-    
+
     def get_event_log(self, entries: int = 100) -> Optional[List[Dict]]:
         sql = event_log.select().limit(entries).all()
         result = self.execute(sql)
 
-        if result is None: return None
+        if result is None:
+            return None
         return result.fetchall()
-    
+
     def fix_bools(self, data: Dict) -> Dict:
-        for k,v in data.items():
+        for k, v in data.items():
             if type(v) == str and v.lower() == "true":
                 data[k] = True
             elif type(v) == str and v.lower() == "false":
                 data[k] = False
-        
+
         return data

core/data/schema/card.py

@@ -3,55 +3,92 @@ from sqlalchemy import Table, Column, UniqueConstraint
 from sqlalchemy.types import Integer, String, Boolean, TIMESTAMP
 from sqlalchemy.sql.schema import ForeignKey
 from sqlalchemy.sql import func
+from sqlalchemy.engine import Row
 
 from core.data.schema.base import BaseData, metadata
 
 aime_card = Table(
-    'aime_card',
+    "aime_card",
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
-    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
     Column("access_code", String(20)),
     Column("created_date", TIMESTAMP, server_default=func.now()),
     Column("last_login_date", TIMESTAMP, onupdate=func.now()),
     Column("is_locked", Boolean, server_default="0"),
     Column("is_banned", Boolean, server_default="0"),
     UniqueConstraint("user", "access_code", name="aime_card_uk"),
-    mysql_charset='utf8mb4'
+    mysql_charset="utf8mb4",
 )
 
+
 class CardData(BaseData):
+    def get_card_by_access_code(self, access_code: str) -> Optional[Row]:
+        sql = aime_card.select(aime_card.c.access_code == access_code)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_card_by_id(self, card_id: int) -> Optional[Row]:
+        sql = aime_card.select(aime_card.c.id == card_id)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def update_access_code(self, old_ac: str, new_ac: str) -> None:
+        sql = aime_card.update(aime_card.c.access_code == old_ac).values(
+            access_code=new_ac
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(
+                f"Failed to change card access code from {old_ac} to {new_ac}"
+            )
+
     def get_user_id_from_card(self, access_code: str) -> Optional[int]:
         """
         Given a 20 digit access code as a string, get the user id associated with that card
         """
-        sql = aime_card.select(aime_card.c.access_code == access_code)
+        card = self.get_card_by_access_code(access_code)
-
+        if card is None:
-        result = self.execute(sql)
+            return None
-        if result is None: return None 
-
-        card = result.fetchone()
-        if card is None: return None
 
         return int(card["user"])
 
-    def get_user_cards(self, aime_id: int) -> Optional[List[Dict]]:
+    def delete_card(self, card_id: int) -> None:
+        sql = aime_card.delete(aime_card.c.id == card_id)
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(f"Failed to delete card with id {card_id}")
+
+    def get_user_cards(self, aime_id: int) -> Optional[List[Row]]:
         """
         Returns all cards owned by a user
         """
         sql = aime_card.select(aime_card.c.user == aime_id)
         result = self.execute(sql)
-        if result is None: return None        
+        if result is None:
+            return None
         return result.fetchall()
 
-    
     def create_card(self, user_id: int, access_code: str) -> Optional[int]:
         """
         Given a aime_user id and a 20 digit access code as a string, create a card and return the ID if successful
         """
         sql = aime_card.insert().values(user=user_id, access_code=access_code)
         result = self.execute(sql)
-        if result is None: return None
+        if result is None:
+            return None
         return result.lastrowid
 
     def to_access_code(self, luid: str) -> str:
@@ -64,4 +101,4 @@ class CardData(BaseData):
         """
         Given a 20 digit access code as a string, return the 16 hex character luid
         """
-        return f'{int(access_code):0{16}x}'
+        return f"{int(access_code):0{16}x}"

core/data/schema/user.py

@@ -1,14 +1,12 @@
 from enum import Enum
-from typing import Dict, Optional
+from typing import Optional, List
-from sqlalchemy import Table, Column, and_
+from sqlalchemy import Table, Column
 from sqlalchemy.types import Integer, String, TIMESTAMP
-from sqlalchemy.sql.schema import ForeignKey
 from sqlalchemy.sql import func
 from sqlalchemy.dialects.mysql import insert
-from sqlalchemy.sql import func, select, Delete
+from sqlalchemy.sql import func, select
-from uuid import uuid4
-from datetime import datetime, timedelta
 from sqlalchemy.engine import Row
+import bcrypt
 
 from core.data.schema.base import BaseData, metadata
 
@@ -19,107 +17,90 @@ aime_user = Table(
     Column("username", String(25), unique=True),
     Column("email", String(255), unique=True),
     Column("password", String(255)),
-    Column("permissions", Integer),    
+    Column("permissions", Integer),
     Column("created_date", TIMESTAMP, server_default=func.now()),
     Column("last_login_date", TIMESTAMP, onupdate=func.now()),
     Column("suspend_expire_time", TIMESTAMP),
-    mysql_charset='utf8mb4'
+    mysql_charset="utf8mb4",
 )
 
-frontend_session = Table(
-    "frontend_session",
-    metadata,
-    Column("id", Integer, primary_key=True, unique=True),
-    Column("user", ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"), nullable=False),
-    Column("ip", String(15)),
-    Column('session_cookie', String(32), nullable=False, unique=True),
-    Column("expires", TIMESTAMP, nullable=False),
-    mysql_charset='utf8mb4'
-)
 
 class PermissionBits(Enum):
     PermUser = 1
     PermMod = 2
     PermSysAdmin = 4
 
-class UserData(BaseData):
-    def create_user(self, id: int = None, username: str = None, email: str = None, password: str = None, permission: int = 1) -> Optional[int]:
-        if email is None:
-            permission = 1
 
+class UserData(BaseData):
+    def create_user(
+        self,
+        id: int = None,
+        username: str = None,
+        email: str = None,
+        password: str = None,
+        permission: int = 1,
+    ) -> Optional[int]:
         if id is None:
             sql = insert(aime_user).values(
-                username=username, 
+                username=username,
-                email=email, 
+                email=email,
-                password=password, 
+                password=password,
-                permissions=permission
+                permissions=permission,
             )
         else:
             sql = insert(aime_user).values(
-                id=id, 
+                id=id,
-                username=username, 
+                username=username,
-                email=email, 
+                email=email,
-                password=password, 
+                password=password,
-                permissions=permission
+                permissions=permission,
             )
 
         conflict = sql.on_duplicate_key_update(
-            username=username, 
+            username=username, email=email, password=password, permissions=permission
-            email=email, 
-            password=password, 
-            permissions=permission
         )
-        
+
         result = self.execute(conflict)
-        if result is None: return None
-        return result.lastrowid
-    
-    def login(self, user_id: int, passwd: bytes = None, ip: str = "0.0.0.0") -> Optional[str]:
-        sql = select(aime_user).where(and_(aime_user.c.id == user_id, aime_user.c.password == passwd))
-
-        result = self.execute(sql)
-        if result is None: return None
-        
-        usr = result.fetchone()
-        if usr is None: return None
-
-        return self.create_session(user_id, ip)
-    
-    def check_session(self, cookie: str, ip: str = "0.0.0.0") -> Optional[Row]:
-        sql = select(frontend_session).where(
-            and_(
-                frontend_session.c.session_cookie == cookie,
-                frontend_session.c.ip == ip
-            )
-        )
-
-        result = self.execute(sql)
-        if result is None: return None        
-        return result.fetchone()
-    
-    def delete_session(self, session_id: int) -> bool:
-        sql = Delete(frontend_session).where(frontend_session.c.id == session_id)
-
-        result = self.execute(sql)
-        if result is None: return False
-        return True
-
-    def create_session(self, user_id: int, ip: str = "0.0.0.0", expires: datetime = datetime.now() + timedelta(days=1)) -> Optional[str]:
-        cookie = uuid4().hex
-
-        sql = insert(frontend_session).values(
-            user = user_id,
-            ip = ip,
-            session_cookie = cookie,
-            expires = expires
-        )
-
-        result = self.execute(sql)
         if result is None:
             return None
-        return cookie
+        return result.lastrowid
+
+    def get_user(self, user_id: int) -> Optional[Row]:
+        sql = select(aime_user).where(aime_user.c.id == user_id)
+        result = self.execute(sql)
+        if result is None:
+            return False
+        return result.fetchone()
+
+    def check_password(self, user_id: int, passwd: bytes = None) -> bool:
+        usr = self.get_user(user_id)
+        if usr is None:
+            return False
+
+        if usr["password"] is None:
+            return False
+
+        return bcrypt.checkpw(passwd, usr["password"].encode())
 
     def reset_autoincrement(self, ai_value: int) -> None:
         # ALTER TABLE isn't in sqlalchemy so we do this the ugly way
         sql = f"ALTER TABLE aime_user AUTO_INCREMENT={ai_value}"
-        self.execute(sql)
+        self.execute(sql)
+
+    def delete_user(self, user_id: int) -> None:
+        sql = aime_user.delete(aime_user.c.id == user_id)
+
+        result = self.execute(sql)
+        if result is None:
+            self.logger.error(f"Failed to delete user with id {user_id}")
+
+    def get_unregistered_users(self) -> List[Row]:
+        """
+        Returns a list of users who have not registered with the webui. They may or may not have cards.
+        """
+        sql = select(aime_user).where(aime_user.c.password == None)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()

core/data/schema/versions/CORE_2_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE `event_log` DROP COLUMN `message`;

core/data/schema/versions/CORE_3_rollback.sql

@@ -0,0 +1,12 @@
+CREATE TABLE `frontend_session` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `user` int(11) NOT NULL,
+  `ip` varchar(15) DEFAULT NULL,
+  `session_cookie` varchar(32) NOT NULL,
+  `expires` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `id` (`id`),
+  UNIQUE KEY `session_cookie` (`session_cookie`),
+  KEY `user` (`user`),
+  CONSTRAINT `frontend_session_ibfk_1` FOREIGN KEY (`user`) REFERENCES `aime_user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4;

core/data/schema/versions/CORE_3_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE `event_log` ADD COLUMN `message` VARCHAR(1000) NOT NULL AFTER `severity`;

core/data/schema/versions/CORE_4_upgrade.sql

@@ -0,0 +1 @@
+DROP TABLE `frontend_session`;

core/data/schema/versions/SBZV_1_rollback.sql

@@ -1,9 +1,9 @@
 SET FOREIGN_KEY_CHECKS=0;
-ALTER TABLE diva_score DROP COLUMN edition;
-ALTER TABLE diva_playlog DROP COLUMN edition;
-
 ALTER TABLE diva_score DROP FOREIGN KEY diva_score_ibfk_1;
 ALTER TABLE diva_score DROP CONSTRAINT diva_score_uk;
 ALTER TABLE diva_score ADD CONSTRAINT diva_score_uk UNIQUE (user, pv_id, difficulty);
 ALTER TABLE diva_score ADD CONSTRAINT diva_score_ibfk_1 FOREIGN KEY (user) REFERENCES aime_user(id) ON DELETE CASCADE;
+
+ALTER TABLE diva_score DROP COLUMN edition;
+ALTER TABLE diva_playlog DROP COLUMN edition;
 SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDDT_3_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE ongeki_profile_data DROP COLUMN lastEmoneyCredit;

core/data/schema/versions/SDDT_4_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE ongeki_profile_data ADD COLUMN lastEmoneyCredit INTEGER DEFAULT 0;

core/data/schema/versions/SDEZ_2_rollback.sql

@@ -0,0 +1,21 @@
+ALTER TABLE mai2_item_card
+CHANGE COLUMN cardId card_id INT NOT NULL AFTER user,
+CHANGE COLUMN cardTypeId card_kind INT NOT NULL,
+CHANGE COLUMN charaId chara_id INT NOT NULL,
+CHANGE COLUMN mapId map_id INT NOT NULL,
+CHANGE COLUMN startDate start_date TIMESTAMP NULL DEFAULT '2018-01-01 00:00:00',
+CHANGE COLUMN endDate end_date TIMESTAMP NULL DEFAULT '2038-01-01 00:00:00';
+
+ALTER TABLE mai2_item_item 
+CHANGE COLUMN itemId item_id INT NOT NULL AFTER user,
+CHANGE COLUMN itemKind item_kind INT NOT NULL,
+CHANGE COLUMN isValid is_valid TINYINT(1) NOT NULL DEFAULT '1';
+
+ALTER TABLE mai2_item_character 
+CHANGE COLUMN characterId character_id INT NOT NULL,
+CHANGE COLUMN useCount use_count INT NOT NULL DEFAULT '0';
+
+ALTER TABLE mai2_item_charge 
+CHANGE COLUMN chargeId charge_id INT NOT NULL,
+CHANGE COLUMN purchaseDate purchase_date TIMESTAMP NOT NULL,
+CHANGE COLUMN validDate valid_date TIMESTAMP NOT NULL;

core/data/schema/versions/SDEZ_3_upgrade.sql

@@ -0,0 +1,21 @@
+ALTER TABLE mai2_item_card
+CHANGE COLUMN card_id cardId INT NOT NULL AFTER user,
+CHANGE COLUMN card_kind cardTypeId INT NOT NULL,
+CHANGE COLUMN chara_id charaId INT NOT NULL,
+CHANGE COLUMN map_id mapId INT NOT NULL,
+CHANGE COLUMN start_date startDate TIMESTAMP NULL DEFAULT '2018-01-01 00:00:00',
+CHANGE COLUMN end_date endDate TIMESTAMP NULL DEFAULT '2038-01-01 00:00:00';
+
+ALTER TABLE mai2_item_item 
+CHANGE COLUMN item_id itemId INT NOT NULL AFTER user,
+CHANGE COLUMN item_kind itemKind INT NOT NULL,
+CHANGE COLUMN is_valid isValid TINYINT(1) NOT NULL DEFAULT '1';
+
+ALTER TABLE mai2_item_character 
+CHANGE COLUMN character_id characterId INT NOT NULL,
+CHANGE COLUMN use_count useCount INT NOT NULL DEFAULT '0';
+
+ALTER TABLE mai2_item_charge 
+CHANGE COLUMN charge_id chargeId INT NOT NULL,
+CHANGE COLUMN purchase_date purchaseDate TIMESTAMP NOT NULL,
+CHANGE COLUMN valid_date validDate TIMESTAMP NOT NULL;

core/frontend.py

@@ -4,17 +4,34 @@ from twisted.web import resource
 from twisted.web.util import redirectTo
 from twisted.web.http import Request
 from logging.handlers import TimedRotatingFileHandler
+from twisted.web.server import Session
+from zope.interface import Interface, Attribute, implementer
+from twisted.python.components import registerAdapter
 import jinja2
 import bcrypt
 
-from core.config import CoreConfig
+from core import CoreConfig, Utils
 from core.data import Data
-from core.utils import Utils
+
+
+class IUserSession(Interface):
+    userId = Attribute("User's ID")
+    current_ip = Attribute("User's current ip address")
+    permissions = Attribute("User's permission level")
+
+
+@implementer(IUserSession)
+class UserSession(object):
+    def __init__(self, session):
+        self.userId = 0
+        self.current_ip = "0.0.0.0"
+        self.permissions = 0
+
 
 class FrontendServlet(resource.Resource):
     def getChild(self, name: bytes, request: Request):
-        self.logger.debug(f"{request.getClientIP()} -> {name.decode()}")
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {name.decode()}")
-        if name == b'':
+        if name == b"":
             return self
         return resource.Resource.getChild(self, name, request)
 
@@ -27,17 +44,24 @@ class FrontendServlet(resource.Resource):
         self.game_list: List[Dict[str, str]] = []
         self.children: Dict[str, Any] = {}
 
-        fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.log_dir, "frontend"), when="d", backupCount=10)
+        fileHandler = TimedRotatingFileHandler(
+            "{0}/{1}.log".format(self.config.server.log_dir, "frontend"),
+            when="d",
+            backupCount=10,
+        )
         fileHandler.setFormatter(log_fmt)
-        
+
         consoleHandler = logging.StreamHandler()
         consoleHandler.setFormatter(log_fmt)
 
         self.logger.addHandler(fileHandler)
         self.logger.addHandler(consoleHandler)
-        
+
         self.logger.setLevel(cfg.frontend.loglevel)
-        coloredlogs.install(level=cfg.frontend.loglevel, logger=self.logger, fmt=log_fmt_str)
+        coloredlogs.install(
+            level=cfg.frontend.loglevel, logger=self.logger, fmt=log_fmt_str
+        )
+        registerAdapter(UserSession, Session, IUserSession)
 
         fe_game = FE_Game(cfg, self.environment)
         games = Utils.get_all_titles()
@@ -49,18 +73,26 @@ class FrontendServlet(resource.Resource):
                     fe_game.putChild(game_dir.encode(), game_fe)
                 except:
                     raise
-        
+
         self.environment.globals["game_list"] = self.game_list
         self.putChild(b"gate", FE_Gate(cfg, self.environment))
         self.putChild(b"user", FE_User(cfg, self.environment))
         self.putChild(b"game", fe_game)
 
-        self.logger.info(f"Ready on port {self.config.frontend.port} serving {len(fe_game.children)} games")
+        self.logger.info(
+            f"Ready on port {self.config.frontend.port} serving {len(fe_game.children)} games"
+        )
 
     def render_GET(self, request):
-        self.logger.debug(f"{request.getClientIP()} -> {request.uri.decode()}")
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.uri.decode()}")
-        template = self.environment.get_template("core/frontend/index.jinja")        
+        template = self.environment.get_template("core/frontend/index.jinja")
-        return template.render(server_name=self.config.server.name, title=self.config.server.name, game_list=self.game_list).encode("utf-16")
+        return template.render(
+            server_name=self.config.server.name,
+            title=self.config.server.name,
+            game_list=self.game_list,
+            sesh=vars(IUserSession(request.getSession())),
+        ).encode("utf-16")
+
 
 class FE_Base(resource.Resource):
     """
@@ -68,63 +100,81 @@ class FE_Base(resource.Resource):
     Initializes the environment, data, logger, config, and sets isLeaf to true
     It is expected that game implementations of this class overwrite many of these
     """
-    isLeaf = True    
+
+    isLeaf = True
+
     def __init__(self, cfg: CoreConfig, environment: jinja2.Environment) -> None:
         self.core_config = cfg
         self.data = Data(cfg)
-        self.logger = logging.getLogger('frontend')
+        self.logger = logging.getLogger("frontend")
         self.environment = environment
         self.nav_name = "nav_name"
 
+
 class FE_Gate(FE_Base):
-    def render_GET(self, request: Request):        
+    def render_GET(self, request: Request):
-        self.logger.debug(f"{request.getClientIP()} -> {request.uri.decode()}")
+        self.logger.debug(f"{Utils.get_ip_addr(request)} -> {request.uri.decode()}")
         uri: str = request.uri.decode()
+
+        sesh = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        if usr_sesh.userId > 0:
+            return redirectTo(b"/user", request)
+
         if uri.startswith("/gate/create"):
             return self.create_user(request)
 
-        if b'e' in request.args:
+        if b"e" in request.args:
             try:
-                err = int(request.args[b'e'][0].decode())
+                err = int(request.args[b"e"][0].decode())
             except:
                 err = 0
 
-        else: err = 0
+        else:
+            err = 0
+
+        template = self.environment.get_template("core/frontend/gate/gate.jinja")
+        return template.render(
+            title=f"{self.core_config.server.name} | Login Gate",
+            error=err,
+            sesh=vars(usr_sesh),
+        ).encode("utf-16")
 
-        template = self.environment.get_template("core/frontend/gate/gate.jinja")        
-        return template.render(title=f"{self.core_config.server.name} | Login Gate", error=err).encode("utf-16")
-    
     def render_POST(self, request: Request):
         uri = request.uri.decode()
-        ip = request.getClientAddress().host
+        ip = Utils.get_ip_addr(request)
-        
+
-        if uri == "/gate/gate.login":            
+        if uri == "/gate/gate.login":
             access_code: str = request.args[b"access_code"][0].decode()
-            passwd: str = request.args[b"passwd"][0]
+            passwd: bytes = request.args[b"passwd"][0]
             if passwd == b"":
                 passwd = None
 
             uid = self.data.card.get_user_id_from_card(access_code)
             if uid is None:
                 return redirectTo(b"/gate?e=1", request)
-                        
+
             if passwd is None:
-                sesh = self.data.user.login(uid, ip=ip)
+                sesh = self.data.user.check_password(uid)
 
                 if sesh is not None:
-                    return redirectTo(f"/gate/create?ac={access_code}".encode(), request)
+                    return redirectTo(
+                        f"/gate/create?ac={access_code}".encode(), request
+                    )
                 return redirectTo(b"/gate?e=1", request)
 
-            salt = bcrypt.gensalt()
+            if not self.data.user.check_password(uid, passwd):
-            hashed = bcrypt.hashpw(passwd, salt)
-            sesh = self.data.user.login(uid, hashed, ip)
-
-            if sesh is None:
                 return redirectTo(b"/gate?e=1", request)
 
-            request.addCookie('session', sesh)            
+            self.logger.info(f"Successful login of user {uid} at {ip}")
+
+            sesh = request.getSession()
+            usr_sesh = IUserSession(sesh)
+            usr_sesh.userId = uid
+            usr_sesh.current_ip = ip
+
             return redirectTo(b"/user", request)
-        
+
         elif uri == "/gate/gate.create":
             access_code: str = request.args[b"access_code"][0].decode()
             username: str = request.args[b"username"][0]
@@ -138,44 +188,56 @@ class FE_Gate(FE_Base):
             salt = bcrypt.gensalt()
             hashed = bcrypt.hashpw(passwd, salt)
 
-            result = self.data.user.create_user(uid, username, email, hashed.decode(), 1)
+            result = self.data.user.create_user(
+                uid, username, email, hashed.decode(), 1
+            )
             if result is None:
                 return redirectTo(b"/gate?e=3", request)
-            
+
-            sesh = self.data.user.login(uid, hashed, ip)
+            if not self.data.user.check_password(uid, passwd.encode()):
-            if sesh is None:
                 return redirectTo(b"/gate", request)
-            request.addCookie('session', sesh)
+
-            
             return redirectTo(b"/user", request)
 
         else:
             return b""
 
     def create_user(self, request: Request):
-        if b'ac' not in request.args or len(request.args[b'ac'][0].decode()) != 20:            
+        if b"ac" not in request.args or len(request.args[b"ac"][0].decode()) != 20:
             return redirectTo(b"/gate?e=2", request)
 
-        ac = request.args[b'ac'][0].decode()
+        ac = request.args[b"ac"][0].decode()
-        
+
-        template = self.environment.get_template("core/frontend/gate/create.jinja")        
+        template = self.environment.get_template("core/frontend/gate/create.jinja")
-        return template.render(title=f"{self.core_config.server.name} | Create User", code=ac).encode("utf-16")
+        return template.render(
+            title=f"{self.core_config.server.name} | Create User",
+            code=ac,
+            sesh={"userId": 0},
+        ).encode("utf-16")
+
 
 class FE_User(FE_Base):
     def render_GET(self, request: Request):
         template = self.environment.get_template("core/frontend/user/index.jinja")
-        return template.render().encode("utf-16")
+
-        if b'session' not in request.cookies:
+        sesh: Session = request.getSession()
+        usr_sesh = IUserSession(sesh)
+        if usr_sesh.userId == 0:
             return redirectTo(b"/gate", request)
 
+        return template.render(
+            title=f"{self.core_config.server.name} | Account", sesh=vars(usr_sesh)
+        ).encode("utf-16")
+
+
 class FE_Game(FE_Base):
     isLeaf = False
     children: Dict[str, Any] = {}
 
     def getChild(self, name: bytes, request: Request):
-        if name == b'':
+        if name == b"":
             return self
         return resource.Resource.getChild(self, name, request)
 
     def render_GET(self, request: Request) -> bytes:
-        return redirectTo(b"/user", request)
+        return redirectTo(b"/user", request)

core/frontend/gate/gate.jinja

@@ -2,10 +2,23 @@
 {% block content %}
 <h1>Gate</h1>
 {% include "core/frontend/widgets/err_banner.jinja" %}
+<style>
+    /* Chrome, Safari, Edge, Opera */
+    input::-webkit-outer-spin-button,
+    input::-webkit-inner-spin-button {
+        -webkit-appearance: none;
+        margin: 0;
+    }
+
+    /* Firefox */
+    input[type=number] {
+        -moz-appearance: textfield;
+    }
+</style>
 <form id="login" style="max-width: 240px; min-width: 10%;" action="/gate/gate.login" method="post">
     <div class="form-group row">
         <label for="access_code">Card Access Code</label><br>
-        <input form="login" class="form-control" name="access_code" id="access_code" type="text" placeholder="00000000000000000000" maxlength="20" required>
+        <input form="login" class="form-control" name="access_code" id="access_code" type="number" placeholder="00000000000000000000" maxlength="20" required>
     </div>
     <div class="form-group row">
         <label for="passwd">Password</label><br>
@@ -14,4 +27,6 @@
     <p></p>
     <input id="submit" class="btn btn-primary" style="display: block; margin: 0 auto;" form="login" type="submit" value="Login">
 </form>
+<h6>*To register for the webui, type in the access code of your card, as shown in a game, and leave the password field blank.</h6>
+<h6>*If you have not registered a card with this server, you cannot create a webui account.</h6>
 {% endblock content %}

core/frontend/widgets/topbar.jinja

@@ -9,5 +9,10 @@
 </div>
 </div>
 <div style="background: #333; color: #f9f9f9; width: 10%; height: 50px; line-height: 50px; text-align: center; float: left;">
+    {% if sesh is defined and sesh["userId"] > 0 %}
+    <a href="/user"><button class="btn btn-primary">Account</button></a>
+    {% else %}
     <a href="/gate"><button class="btn btn-primary">Gate</button></a>
+    {% endif %}
+    
 </div>

core/mucha.py

@@ -1,4 +1,4 @@
-from typing import Dict, Any, Optional
+from typing import Dict, Any, Optional, List
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
 from twisted.web import resource
@@ -6,59 +6,101 @@ from twisted.web.http import Request
 from datetime import datetime
 import pytz
 
-from core.config import CoreConfig
+from core import CoreConfig
+from core.utils import Utils
+
 
 class MuchaServlet:
-    def __init__(self, cfg: CoreConfig) -> None:        
+    def __init__(self, cfg: CoreConfig, cfg_dir: str) -> None:
         self.config = cfg
+        self.config_dir = cfg_dir
+        self.mucha_registry: List[str] = []
 
-        self.logger = logging.getLogger('mucha')
+        self.logger = logging.getLogger("mucha")
         log_fmt_str = "[%(asctime)s] Mucha | %(levelname)s | %(message)s"
         log_fmt = logging.Formatter(log_fmt_str)
-        
+
-        fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.log_dir, "mucha"), when="d", backupCount=10)
+        fileHandler = TimedRotatingFileHandler(
+            "{0}/{1}.log".format(self.config.server.log_dir, "mucha"),
+            when="d",
+            backupCount=10,
+        )
         fileHandler.setFormatter(log_fmt)
-        
+
         consoleHandler = logging.StreamHandler()
         consoleHandler.setFormatter(log_fmt)
 
         self.logger.addHandler(fileHandler)
         self.logger.addHandler(consoleHandler)
-        
+
         self.logger.setLevel(logging.INFO)
         coloredlogs.install(level=logging.INFO, logger=self.logger, fmt=log_fmt_str)
 
+        all_titles = Utils.get_all_titles()
+
+        for _, mod in all_titles.items():
+            if hasattr(mod, "index") and hasattr(mod.index, "get_mucha_info"):
+                enabled, game_cd = mod.index.get_mucha_info(
+                    self.config, self.config_dir
+                )
+                if enabled:
+                    self.mucha_registry.append(game_cd)
+
+        self.logger.info(
+            f"Serving {len(self.mucha_registry)} games"
+        )
+
     def handle_boardauth(self, request: Request, _: Dict) -> bytes:
         req_dict = self.mucha_preprocess(request.content.getvalue())
+        client_ip = Utils.get_ip_addr(request)
+
         if req_dict is None:
-            self.logger.error(f"Error processing mucha request {request.content.getvalue()}")
+            self.logger.error(
+                f"Error processing mucha request {request.content.getvalue()}"
+            )
             return b""
 
         req = MuchaAuthRequest(req_dict)
         self.logger.debug(f"Mucha request {vars(req)}")
+        self.logger.info(
+            f"Boardauth request from {client_ip} for {req.gameVer}"
+        )
 
-        if self.config.server.is_develop:
+        if req.gameCd not in self.mucha_registry:
-            resp = MuchaAuthResponse(mucha_url=f"{self.config.mucha.hostname}:{self.config.mucha.port}")
+            self.logger.warn(f"Unknown gameCd {req.gameCd}")
-        else:
+            return b""
-            resp = MuchaAuthResponse(mucha_url=f"{self.config.mucha.hostname}")
+
+        # TODO: Decrypt S/N
+
+        resp = MuchaAuthResponse(
+            f"{self.config.mucha.hostname}{':' + str(self.config.allnet.port) if self.config.server.is_develop else ''}"
+        )
 
         self.logger.debug(f"Mucha response {vars(resp)}")
 
         return self.mucha_postprocess(vars(resp))
-    
+
     def handle_updatecheck(self, request: Request, _: Dict) -> bytes:
         req_dict = self.mucha_preprocess(request.content.getvalue())
+        client_ip = Utils.get_ip_addr(request)
+
         if req_dict is None:
-            self.logger.error(f"Error processing mucha request {request.content.getvalue()}")
+            self.logger.error(
+                f"Error processing mucha request {request.content.getvalue()}"
+            )
             return b""
 
         req = MuchaUpdateRequest(req_dict)
         self.logger.debug(f"Mucha request {vars(req)}")
+        self.logger.info(
+            f"Updatecheck request from {client_ip} for {req.gameVer}"
+        )
 
-        if self.config.server.is_develop:
+        if req.gameCd not in self.mucha_registry:
-            resp = MuchaUpdateResponse(mucha_url=f"{self.config.mucha.hostname}:{self.config.mucha.port}")
+            self.logger.warn(f"Unknown gameCd {req.gameCd}")
-        else:
+            return b""
-            resp = MuchaUpdateResponse(mucha_url=f"{self.config.mucha.hostname}")
+
+        resp = MuchaUpdateResponseStub(req.gameVer)
 
         self.logger.debug(f"Mucha response {vars(resp)}")
 
@@ -67,14 +109,14 @@ class MuchaServlet:
     def mucha_preprocess(self, data: bytes) -> Optional[Dict]:
         try:
             ret: Dict[str, Any] = {}
-            
+
-            for x in data.decode().split('&'):
+            for x in data.decode().split("&"):
-                kvp = x.split('=')
+                kvp = x.split("=")
                 if len(kvp) == 2:
                     ret[kvp[0]] = kvp[1]
 
             return ret
-        
+
         except:
             self.logger.error(f"Error processing mucha request {data}")
             return None
@@ -82,7 +124,7 @@ class MuchaServlet:
     def mucha_postprocess(self, data: dict) -> Optional[bytes]:
         try:
             urlencode = ""
-            for k,v in data.items():
+            for k, v in data.items():
                 urlencode += f"{k}={v}&"
 
             return urlencode.encode()
@@ -91,35 +133,39 @@ class MuchaServlet:
             self.logger.error("Error processing mucha response")
             return None
 
-class MuchaAuthRequest():
-    def __init__(self, request: Dict) -> None:
-        self.gameVer = "" if "gameVer" not in request else request["gameVer"]
-        self.sendDate = "" if "sendDate" not in request else request["sendDate"]
-        self.serialNum = "" if "serialNum" not in request else request["serialNum"]
-        self.gameCd = "" if "gameCd" not in request else request["gameCd"]
-        self.boardType = "" if "boardType" not in request else request["boardType"]
-        self.boardId = "" if "boardId" not in request else request["boardId"]
-        self.placeId = "" if "placeId" not in request else request["placeId"]
-        self.storeRouterIp = "" if "storeRouterIp" not in request else request["storeRouterIp"]
-        self.countryCd = "" if "countryCd" not in request else request["countryCd"]
-        self.useToken = "" if "useToken" not in request else request["useToken"]
-        self.allToken = "" if "allToken" not in request else request["allToken"]
 
-class MuchaAuthResponse():
+class MuchaAuthRequest:
-    def __init__(self, mucha_url: str = "localhost") -> None:
+    def __init__(self, request: Dict) -> None:
-        self.RESULTS = "001"        
+        # gameCd + boardType + countryCd + version
+        self.gameVer = request.get("gameVer", "")
+        self.sendDate = request.get("sendDate", "")  # %Y%m%d
+        self.serialNum = request.get("serialNum", "")
+        self.gameCd = request.get("gameCd", "")
+        self.boardType = request.get("boardType", "")
+        self.boardId = request.get("boardId", "")
+        self.mac = request.get("mac", "")
+        self.placeId = request.get("placeId", "")
+        self.storeRouterIp = request.get("storeRouterIp", "")
+        self.countryCd = request.get("countryCd", "")
+        self.useToken = request.get("useToken", "")
+        self.allToken = request.get("allToken", "")
+
+
+class MuchaAuthResponse:
+    def __init__(self, mucha_url: str) -> None:
+        self.RESULTS = "001"
         self.AUTH_INTERVAL = "86400"
         self.SERVER_TIME = datetime.strftime(datetime.now(), "%Y%m%d%H%M")
         self.UTC_SERVER_TIME = datetime.strftime(datetime.now(pytz.UTC), "%Y%m%d%H%M")
 
-        self.CHARGE_URL = f"https://{mucha_url}/charge/"        
+        self.CHARGE_URL = f"https://{mucha_url}/charge/"
         self.FILE_URL = f"https://{mucha_url}/file/"
         self.URL_1 = f"https://{mucha_url}/url1/"
         self.URL_2 = f"https://{mucha_url}/url2/"
         self.URL_3 = f"https://{mucha_url}/url3/"
-        
+
-        self.PLACE_ID = "JPN123"        
+        self.PLACE_ID = "JPN123"
-        self.COUNTRY_CD = "JPN"        
+        self.COUNTRY_CD = "JPN"
         self.SHOP_NAME = "TestShop!"
         self.SHOP_NICKNAME = "TestShop"
         self.AREA_0 = "008"
@@ -130,7 +176,7 @@ class MuchaAuthResponse():
         self.AREA_FULL_1 = ""
         self.AREA_FULL_2 = ""
         self.AREA_FULL_3 = ""
-        
+
         self.SHOP_NAME_EN = "TestShop!"
         self.SHOP_NICKNAME_EN = "TestShop"
         self.AREA_0_EN = "008"
@@ -142,24 +188,26 @@ class MuchaAuthResponse():
         self.AREA_FULL_2_EN = ""
         self.AREA_FULL_3_EN = ""
 
-        self.PREFECTURE_ID = "1"        
+        self.PREFECTURE_ID = "1"
         self.EXPIRATION_DATE = "null"
         self.USE_TOKEN = "0"
         self.CONSUME_TOKEN = "0"
         self.DONGLE_FLG = "1"
         self.FORCE_BOOT = "0"
 
-class MuchaUpdateRequest():
-    def __init__(self, request: Dict) -> None:
-        self.gameVer = "" if "gameVer" not in request else request["gameVer"]
-        self.gameCd = "" if "gameCd" not in request else request["gameCd"]
-        self.serialNum = "" if "serialNum" not in request else request["serialNum"]
-        self.countryCd = "" if "countryCd" not in request else request["countryCd"]
-        self.placeId = "" if "placeId" not in request else request["placeId"]
-        self.storeRouterIp = "" if "storeRouterIp" not in request else request["storeRouterIp"]
 
-class MuchaUpdateResponse():
+class MuchaUpdateRequest:
-    def __init__(self, game_ver: str = "PKFN0JPN01.01", mucha_url: str = "localhost") -> None:
+    def __init__(self, request: Dict) -> None:
+        self.gameVer = request.get("gameVer", "")
+        self.gameCd = request.get("gameCd", "")
+        self.serialNum = request.get("serialNum", "")
+        self.countryCd = request.get("countryCd", "")
+        self.placeId = request.get("placeId", "")
+        self.storeRouterIp = request.get("storeRouterIp", "")
+
+
+class MuchaUpdateResponse:
+    def __init__(self, game_ver: str, mucha_url: str) -> None:
         self.RESULTS = "001"
         self.UPDATE_VER_1 = game_ver
         self.UPDATE_URL_1 = f"https://{mucha_url}/updUrl1/"
@@ -171,3 +219,11 @@ class MuchaUpdateResponse():
         self.COM_SIZE_1 = "0"
         self.COM_TIME_1 = "0"
         self.LAN_INFO_SIZE_1 = "0"
+        self.USER_ID = ""
+        self.PASSWORD = ""
+
+
+class MuchaUpdateResponseStub:
+    def __init__(self, game_ver: str) -> None:
+        self.RESULTS = "001"
+        self.UPDATE_VER_1 = game_ver

core/title.py

@@ -7,8 +7,9 @@ from core.config import CoreConfig
 from core.data import Data
 from core.utils import Utils
 
-class TitleServlet():
+
-    def __init__(self, core_cfg: CoreConfig, cfg_folder: str):        
+class TitleServlet:
+    def __init__(self, core_cfg: CoreConfig, cfg_folder: str):
         super().__init__()
         self.config = core_cfg
         self.config_folder = cfg_folder
@@ -18,36 +19,57 @@ class TitleServlet():
         self.logger = logging.getLogger("title")
         if not hasattr(self.logger, "initialized"):
             log_fmt_str = "[%(asctime)s] Title | %(levelname)s | %(message)s"
-            log_fmt = logging.Formatter(log_fmt_str)        
+            log_fmt = logging.Formatter(log_fmt_str)
 
-            fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(self.config.server.log_dir, "title"), when="d", backupCount=10)
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.config.server.log_dir, "title"),
+                when="d",
+                backupCount=10,
+            )
             fileHandler.setFormatter(log_fmt)
-            
+
             consoleHandler = logging.StreamHandler()
             consoleHandler.setFormatter(log_fmt)
 
             self.logger.addHandler(fileHandler)
             self.logger.addHandler(consoleHandler)
-            
+
             self.logger.setLevel(core_cfg.title.loglevel)
-            coloredlogs.install(level=core_cfg.title.loglevel, logger=self.logger, fmt=log_fmt_str)
+            coloredlogs.install(
+                level=core_cfg.title.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
             self.logger.initialized = True
-        
+
         plugins = Utils.get_all_titles()
-        
+
         for folder, mod in plugins.items():
             if hasattr(mod, "game_codes") and hasattr(mod, "index"):
-                handler_cls = mod.index(self.config, self.config_folder)
+                should_call_setup = True
-                if hasattr(handler_cls, "setup"):
+
-                    handler_cls.setup()
+                if hasattr(mod.index, "get_allnet_info"):
-                
+                    for code in mod.game_codes:
-                for code in mod.game_codes:
+                        enabled, _, _ = mod.index.get_allnet_info(
-                    self.title_registry[code] = handler_cls
+                            code, self.config, self.config_folder
-            
+                        )
+
+                        if enabled:
+                            handler_cls = mod.index(self.config, self.config_folder)
+
+                            if hasattr(handler_cls, "setup") and should_call_setup:
+                                handler_cls.setup()
+                                should_call_setup = False
+
+                            self.title_registry[code] = handler_cls
+
+                else:
+                    self.logger.warn(f"Game {folder} has no get_allnet_info")
+
             else:
                 self.logger.error(f"{folder} missing game_code or index in __init__.py")
-        
+
-        self.logger.info(f"Serving {len(self.title_registry)} game codes on port {core_cfg.title.port}")
+        self.logger.info(
+            f"Serving {len(self.title_registry)} game codes {'on port ' + str(core_cfg.title.port) if core_cfg.title.port > 0 else ''}"
+        )
 
     def render_GET(self, request: Request, endpoints: dict) -> bytes:
         code = endpoints["game"]
@@ -55,7 +77,7 @@ class TitleServlet():
             self.logger.warn(f"Unknown game code {code}")
             request.setResponseCode(404)
             return b""
-        
+
         index = self.title_registry[code]
         if not hasattr(index, "render_GET"):
             self.logger.warn(f"{code} does not dispatch GET")
@@ -63,18 +85,20 @@ class TitleServlet():
             return b""
 
         return index.render_GET(request, endpoints["version"], endpoints["endpoint"])
-    
+
     def render_POST(self, request: Request, endpoints: dict) -> bytes:
         code = endpoints["game"]
         if code not in self.title_registry:
             self.logger.warn(f"Unknown game code {code}")
             request.setResponseCode(404)
             return b""
-        
+
         index = self.title_registry[code]
         if not hasattr(index, "render_POST"):
             self.logger.warn(f"{code} does not dispatch POST")
             request.setResponseCode(405)
             return b""
 
-        return index.render_POST(request, int(endpoints["version"]), endpoints["endpoint"])
+        return index.render_POST(
+            request, int(endpoints["version"]), endpoints["endpoint"]
+        )

core/utils.py

@@ -1,16 +1,18 @@
 from typing import Dict, Any
 from types import ModuleType
+from twisted.web.http import Request
 import logging
 import importlib
 from os import walk
 
+
 class Utils:
     @classmethod
     def get_all_titles(cls) -> Dict[str, ModuleType]:
         ret: Dict[str, Any] = {}
 
         for root, dirs, files in walk("titles"):
-            for dir in dirs: 
+            for dir in dirs:
                 if not dir.startswith("__"):
                     try:
                         mod = importlib.import_module(f"titles.{dir}")
@@ -20,3 +22,7 @@ class Utils:
                         logging.getLogger("core").error(f"get_all_titles: {dir} - {e}")
                         raise
             return ret
+    
+    @classmethod
+    def get_ip_addr(cls, req: Request) -> str:
+        return req.getAllHeaders()[b"x-forwarded-for"].decode() if b"x-forwarded-for" in req.getAllHeaders() else req.getClientAddress().host

dbutils.py

@@ -2,22 +2,53 @@ import yaml
 import argparse
 from core.config import CoreConfig
 from core.data import Data
+from os import path, mkdir, access, W_OK
 
-if __name__=='__main__':
+if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Database utilities")
-    parser.add_argument("--config", "-c", type=str, help="Config folder to use", default="config")
+    parser.add_argument(
-    parser.add_argument("--version", "-v", type=str, help="Version of the database to upgrade/rollback to")
+        "--config", "-c", type=str, help="Config folder to use", default="config"
-    parser.add_argument("--game", "-g", type=str, help="Game code of the game who's schema will be updated/rolled back. Ex. SDFE")
+    )
-    parser.add_argument("action", type=str, help="DB Action, create, recreate, upgrade, or rollback")
+    parser.add_argument(
+        "--version",
+        "-v",
+        type=str,
+        help="Version of the database to upgrade/rollback to",
+    )
+    parser.add_argument(
+        "--game",
+        "-g",
+        type=str,
+        help="Game code of the game who's schema will be updated/rolled back. Ex. SDFE",
+    )
+    parser.add_argument("--email", "-e", type=str, help="Email for the new user")
+    parser.add_argument("--old_ac", "-o", type=str, help="Access code to transfer from")
+    parser.add_argument("--new_ac", "-n", type=str, help="Access code to transfer to")
+    parser.add_argument("--force", "-f", type=bool, help="Force the action to happen")
+    parser.add_argument(
+        "action", type=str, help="DB Action, create, recreate, upgrade, or rollback"
+    )
     args = parser.parse_args()
 
     cfg = CoreConfig()
-    cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
+    if path.exists(f"{args.config}/core.yaml"):
+        cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
+    
+    if not path.exists(cfg.server.log_dir):
+        mkdir(cfg.server.log_dir)
+    
+    if not access(cfg.server.log_dir, W_OK):
+        print(
+            f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+        )
+        exit(1)
+
     data = Data(cfg)
+    
 
     if args.action == "create":
         data.create_database()
-        
+
     elif args.action == "recreate":
         data.recreate_database()
 
@@ -28,9 +59,21 @@ if __name__=='__main__':
 
         if args.game is None:
             data.logger.info("No game set, upgrading core schema")
-            data.migrate_database("CORE", int(args.version))
+            data.migrate_database("CORE", int(args.version), args.action)
 
         else:
             data.migrate_database(args.game, int(args.version), args.action)
 
+    elif args.action == "autoupgrade":
+        data.autoupgrade()
+
+    elif args.action == "create-owner":
+        data.create_owner(args.email)
+
+    elif args.action == "migrate-card":
+        data.migrate_card(args.old_ac, args.new_ac, args.force)
+
+    elif args.action == "cleanup":
+        data.delete_hanging_users()
+
     data.logger.info("Done")

docs/game_specific_info.md

@@ -0,0 +1,351 @@
+# ARTEMiS Games Documentation
+
+Below are all supported games with supported version ids in order to use
+the corresponding importer and database upgrades.
+
+**Important: The described database upgrades are only required if you are using an old database schema, f.e. still
+using the megaime database. Clean installations always create the latest database structure!**
+
+# Table of content
+
+- [Supported Games](#supported-games)
+    - [Chunithm](#chunithm)
+    - [crossbeats REV.](#crossbeats-rev)
+    - [maimai DX](#maimai-dx)
+    - [O.N.G.E.K.I.](#o-n-g-e-k-i)
+    - [Card Maker](#card-maker)
+    - [WACCA](#wacca)
+
+
+# Supported Games
+
+Games listed below have been tested and confirmed working.
+
+## Chunithm
+
+### SDBT
+
+| Version ID | Version Name       |
+|------------|--------------------|
+| 0          | Chunithm           |
+| 1          | Chunithm+          |
+| 2          | Chunithm Air       |
+| 3          | Chunithm Air +     |
+| 4          | Chunithm Star      |
+| 5          | Chunithm Star +    |
+| 6          | Chunithm Amazon    |
+| 7          | Chunithm Amazon +  |
+| 8          | Chunithm Crystal   |
+| 9          | Chunithm Crystal + |
+| 10         | Chunithm Paradise  |
+
+### SDHD/SDBT
+
+| Version ID | Version Name    |
+|------------|-----------------|
+| 11         | Chunithm New!!  |
+| 12         | Chunithm New!!+ |
+
+
+### Importer
+
+In order to use the importer locate your game installation folder and execute:
+
+```shell
+python read.py --series SDBT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+```
+
+The importer for Chunithm will import: Events, Music, Charge Items and Avatar Accesories.
+
+### Database upgrade
+
+Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see
+which version is the latest, f.e. `SDBT_3_upgrade.sql`. In order to upgrade to version 3 in this case you need to
+perform all previous updates as well:
+
+```shell
+python dbutils.py --game SDBT --version 2 upgrade
+python dbutils.py --game SDBT --version 3 upgrade
+```
+
+## crossbeats REV.
+
+### SDCA
+
+| Version ID | Version Name                       |
+|------------|------------------------------------|
+| 0          | crossbeats REV.                    |
+| 1          | crossbeats REV. SUNRISE            |
+| 2          | crossbeats REV. SUNRISE S2         |
+| 3          | crossbeats REV. SUNRISE S2 Omnimix |
+
+### Importer
+
+In order to use the importer you need to use the provided `Export.csv` file:
+
+```shell
+python read.py --series SDCA --version <version ID> --binfolder titles/cxb/data
+```
+
+The importer for crossbeats REV. will import Music.
+
+### Config
+
+Config file is located in `config/cxb.yaml`.
+
+| Option                 | Info                                                       |
+|------------------------|------------------------------------------------------------|
+| `hostname`             | Requires a proper `hostname` (not localhost!) to run       |
+| `ssl_enable`           | Enables/Disables the use of the `ssl_cert` and `ssl_key`   |
+| `port`                 | Set your unsecure port number                              |
+| `port_secure`          | Set your secure/SSL port number                            |
+| `ssl_cert`, `ssl_key`  | Enter your SSL certificate (requires not self signed cert) |
+
+
+## maimai DX
+
+### SDEZ
+
+| Version ID | Version Name            |
+|------------|-------------------------|
+| 0          | maimai DX               |
+| 1          | maimai DX PLUS          |
+| 2          | maimai DX Splash        |
+| 3          | maimai DX Splash PLUS   |
+| 4          | maimai DX Universe      |
+| 5          | maimai DX Universe PLUS |
+
+### Importer
+
+In order to use the importer locate your game installation folder and execute:
+
+```shell
+python read.py --series SDEZ --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+```
+
+The importer for maimai DX will import Events, Music and Tickets.
+
+**NOTE: It is required to use the importer because the game will
+crash without it!**
+
+### Database upgrade
+
+Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDEZ_2_upgrade.sql`. In order to upgrade to version 2 in this case you need to perform all previous updates as well:
+
+```shell
+python dbutils.py --game SDEZ --version 2 upgrade
+```
+
+## Hatsune Miku Project Diva
+
+### SBZV
+
+| Version ID | Version Name                    |
+|------------|---------------------------------|
+| 0          | Project Diva Arcade             |
+| 1          | Project Diva Arcade Future Tone |
+
+
+### Importer
+
+In order to use the importer locate your game installation folder and execute:
+
+```shell
+python read.py --series SBZV --version <version ID> --binfolder /path/to/game/data/diva --optfolder /path/to/game/data/diva/mdata
+```
+
+The importer for Project Diva Arcade will all required data in order to use
+the Shop, Modules and Customizations.
+
+### Config
+
+Config file is located in `config/diva.yaml`.
+
+| Option               | Info                                                                                            |
+|----------------------|-------------------------------------------------------------------------------------------------|
+| `unlock_all_modules` | Unlocks all modules (costumes) by default, if set to `False` all modules need to be purchased   |
+| `unlock_all_items`   | Unlocks all items (customizations) by default, if set to `False` all items need to be purchased |
+
+
+### Database upgrade
+
+Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see
+which version is the latest, f.e. `SBZV_4_upgrade.sql`. In order to upgrade to version 4 in this case you need to
+perform all previous updates as well:
+
+```shell
+python dbutils.py --game SBZV --version 2 upgrade
+python dbutils.py --game SBZV --version 3 upgrade
+python dbutils.py --game SBZV --version 4 upgrade
+```
+
+## O.N.G.E.K.I.
+
+### SDDT
+
+| Version ID | Version Name               |
+|------------|----------------------------|
+| 0          | O.N.G.E.K.I.               |
+| 1          | O.N.G.E.K.I. +             |
+| 2          | O.N.G.E.K.I. Summer        |
+| 3          | O.N.G.E.K.I. Summer +      |
+| 4          | O.N.G.E.K.I. Red           |
+| 5          | O.N.G.E.K.I. Red +         |
+| 6          | O.N.G.E.K.I. Bright        |
+| 7          | O.N.G.E.K.I. Bright Memory |
+
+
+### Importer
+
+In order to use the importer locate your game installation folder and execute:
+
+```shell
+python read.py --series SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+```
+
+The importer for O.N.G.E.K.I. will all all Cards, Music and Events.
+
+**NOTE: The Importer is required for Card Maker.**
+
+### Config
+
+Config file is located in `config/ongeki.yaml`.
+
+| Option           | Info                                                                                                           |
+|------------------|----------------------------------------------------------------------------------------------------------------|
+| `enabled_gachas` | Enter all gacha IDs for Card Maker to work, other than default may not work due to missing cards added to them |
+
+Note: 1149 and higher are only for Card Maker 1.35 and higher and will be ignored on lower versions.
+
+### Database upgrade
+
+Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see
+which version is the latest, f.e. `SDDT_4_upgrade.sql`. In order to upgrade to version 4 in this case you need to
+perform all previous updates as well:
+
+```shell
+python dbutils.py --game SDDT --version 2 upgrade
+python dbutils.py --game SDDT --version 3 upgrade
+python dbutils.py --game SDDT --version 4 upgrade
+```
+
+## Card Maker
+
+### SDED
+
+| Version ID | Version Name    |
+|------------|-----------------|
+| 0          | Card Maker 1.34 |
+| 1          | Card Maker 1.35 |
+
+
+### Support status
+
+* Card Maker 1.34:
+  * Chunithm New!!: Yes
+  * maimai DX Universe: Yes
+  * O.N.G.E.K.I. Bright: Yes
+
+* Card Maker 1.35:
+  * Chunithm New!!+: Yes
+  * maimai DX Universe PLUS: Yes
+  * O.N.G.E.K.I. Bright Memory: Yes
+
+
+### Importer
+
+In order to use the importer you need to use the provided `.csv` files (which are required for O.N.G.E.K.I.) and the
+option folders:
+
+```shell
+python read.py --series SDED --version <version ID> --binfolder titles/cm/cm_data --optfolder /path/to/cardmaker/option/folder
+```
+
+**If you haven't already executed the O.N.G.E.K.I. importer, make sure you import all cards!**
+
+```shell
+python read.py --series SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+```
+
+Also make sure to import all maimai and Chunithm data as well:
+
+```shell
+python read.py --series SDED --version <version ID> --binfolder /path/to/cardmaker/CardMaker_Data
+```
+
+The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai/Chunithm) and the hardcoded
+Cards for each Gacha (O.N.G.E.K.I. only).
+
+**NOTE: Without executing the importer Card Maker WILL NOT work!**
+
+
+### O.N.G.E.K.I. Gachas
+
+Gacha "無料ガチャ" can only pull from the free cards with the following probabilities: 94%: R, 5% SR and 1% chance of
+getting an SSR card
+
+Gacha "無料ガチャ（SR確定）" can only pull from free SR cards with prob: 92% SR and 8% chance of getting an SSR card
+
+Gacha "レギュラーガチャ" can pull from every card added to ongeki_static_cards with the following prob: 77% R, 20% SR
+and 3% chance of getting an SSR card
+
+All other (limited) gachas can pull from every card added to ongeki_static_cards but with the promoted cards
+(click on the green button under the banner) having a 10 times higher chance to get pulled
+
+### Chunithm Gachas
+
+All cards in Chunithm (basically just the characters) have the same rarity to it just pulls randomly from all cards
+from a given gacha but made sure you cannot pull the same card twice in the same 5 times gacha roll.
+
+### Notes
+
+Card Maker 1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35 will only load an O.N.G.E.K.I.
+Bright Memory profile (1.35).
+The gachas inside the `ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
+Gacha IDs up to 1140 will be loaded for CM 1.34 and all gachas will be loaded for CM 1.35.
+
+**NOTE: There is currently no way to load/use the (printed) maimai DX cards!**
+
+## WACCA
+
+### SDFE
+
+| Version ID | Version Name  |
+|------------|---------------|
+| 0          | WACCA         |
+| 1          | WACCA S       |
+| 2          | WACCA Lily    |
+| 3          | WACCA Lily R  |
+| 4          | WACCA Reverse |
+
+
+### Importer
+
+In order to use the importer locate your game installation folder and execute:
+
+```shell
+python read.py --series SDFE --version <version ID> --binfolder /path/to/game/WindowsNoEditor/Mercury/Content
+```
+
+The importer for WACCA will import all Music data.
+
+### Config
+
+Config file is located in `config/wacca.yaml`.
+
+| Option             | Info                                                                        |
+|--------------------|-----------------------------------------------------------------------------|
+| `always_vip`       | Enables/Disables VIP, if disabled it needs to be purchased manually in game |
+| `infinite_tickets` | Always set the "unlock expert" tickets to 5                                 |
+| `infinite_wp`      | Sets the user WP to `999999`                                                |
+| `enabled_gates`    | Enter all gate IDs which should be enabled in game                          |
+
+
+### Database upgrade
+
+Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDFE_3_upgrade.sql`. In order to upgrade to version 3 in this case you need to perform all previous updates as well:
+
+```shell
+python dbutils.py --game SDFE --version 2 upgrade
+python dbutils.py --game SDFE --version 3 upgrade
+```

example_config/cardmaker.yaml

@@ -0,0 +1,3 @@
+server:
+  enable: True
+  loglevel: "info"

example_config/core.yaml

@@ -48,6 +48,3 @@ mucha:
   enable: False
   hostname: "localhost"
   loglevel: "info"
-  port: 8444
-  ssl_key: "cert/server.key"
-  ssl_cert: "cert/server.pem"

example_config/nginx_example.conf

@@ -4,6 +4,8 @@ server {
 	server_name naominet.jp;
 	
 	location / {
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_pass_request_headers on;
 		proxy_pass http://localhost:8000/;
 	}
 }
@@ -14,6 +16,8 @@ server {
 	server_name your.hostname.here;
 
 	location / {
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_pass_request_headers on;
 		proxy_pass http://localhost:8080/;
 	}
 }
@@ -75,6 +79,8 @@ server {
 	ssl_prefer_server_ciphers off;
 
 	location / {
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_pass_request_headers on;
 		proxy_pass http://localhost:8080/;
 	}
 }
@@ -95,6 +101,8 @@ server {
 	ssl_prefer_server_ciphers off;
 
 	location / {
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_pass_request_headers on;
 		proxy_pass http://localhost:8080/SDBT/104/;
 	}
 }
@@ -131,6 +139,8 @@ server {
     add_header Strict-Transport-Security "max-age=63072000" always;
 
 	location / {
+		proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+		proxy_pass_request_headers on;
 		proxy_pass http://localhost:8090/;
 	}
 }

example_config/ongeki.yaml

@@ -1,3 +1,31 @@
 server:
   enable: True
   loglevel: "info"
+
+gachas:
+  enabled_gachas:
+    - 1011
+    - 1012
+    - 1043
+    - 1067
+    - 1068
+    - 1069
+    - 1070
+    - 1071
+    - 1072
+    - 1073
+    - 1074
+    - 1075
+    - 1076
+    - 1077
+    - 1081
+    - 1085
+    - 1089
+    - 1104
+    - 1111
+    - 1135
+    # can be used for Card Maker 1.35 and up, else will be ignored
+    - 1149
+    - 1156
+    - 1163
+    - 1164

example_config/pokken.yaml

@@ -1,7 +1,8 @@
 server:
+  hostname: "localhost"
   enable: True
   loglevel: "info"
   port: 9000
-  port_matching: 9001
+  port_stun: 9001
-  ssl_cert: cert/pokken.crt
+  port_turn: 9002
-  ssl_key: cert/pokken.key
+  port_admission: 9003

index.py

@@ -12,6 +12,7 @@ from twisted.internet import reactor, endpoints
 from twisted.web.http import Request
 from routes import Mapper
 
+
 class HttpDispatcher(resource.Resource):
     def __init__(self, cfg: CoreConfig, config_dir: str):
         super().__init__()
@@ -20,130 +21,222 @@ class HttpDispatcher(resource.Resource):
         self.map_get = Mapper()
         self.map_post = Mapper()
         self.logger = logging.getLogger("core")
-        
+
         self.allnet = AllnetServlet(cfg, config_dir)
         self.title = TitleServlet(cfg, config_dir)
-        self.mucha = MuchaServlet(cfg)
+        self.mucha = MuchaServlet(cfg, config_dir)
 
-        self.map_post.connect('allnet_ping', '/naomitest.html', controller="allnet", action='handle_naomitest', conditions=dict(method=['GET']))
+        self.map_post.connect(
-        self.map_post.connect('allnet_poweron', '/sys/servlet/PowerOn', controller="allnet", action='handle_poweron', conditions=dict(method=['POST']))
+            "allnet_ping",
-        self.map_post.connect('allnet_downloadorder', '/sys/servlet/DownloadOrder', controller="allnet", action='handle_dlorder', conditions=dict(method=['POST']))
+            "/naomitest.html",
-        self.map_post.connect('allnet_billing', '/request', controller="allnet", action='handle_billing_request', conditions=dict(method=['POST']))
+            controller="allnet",
-        self.map_post.connect('allnet_billing', '/request/', controller="allnet", action='handle_billing_request', conditions=dict(method=['POST']))
+            action="handle_naomitest",
+            conditions=dict(method=["GET"]),
+        )
+        self.map_post.connect(
+            "allnet_poweron",
+            "/sys/servlet/PowerOn",
+            controller="allnet",
+            action="handle_poweron",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "allnet_downloadorder",
+            "/sys/servlet/DownloadOrder",
+            controller="allnet",
+            action="handle_dlorder",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "allnet_billing",
+            "/request",
+            controller="allnet",
+            action="handle_billing_request",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "allnet_billing",
+            "/request/",
+            controller="allnet",
+            action="handle_billing_request",
+            conditions=dict(method=["POST"]),
+        )
 
-        self.map_post.connect('mucha_boardauth', '/mucha/boardauth.do', controller="mucha", action='handle_boardauth', conditions=dict(method=['POST']))
+        self.map_post.connect(
-        self.map_post.connect('mucha_updatacheck', '/mucha/updatacheck.do', controller="mucha", action='handle_updatacheck', conditions=dict(method=['POST']))
+            "mucha_boardauth",
+            "/mucha/boardauth.do",
+            controller="mucha",
+            action="handle_boardauth",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "mucha_updatacheck",
+            "/mucha/updatacheck.do",
+            controller="mucha",
+            action="handle_updatecheck",
+            conditions=dict(method=["POST"]),
+        )
 
-        self.map_get.connect("title_get", "/{game}/{version}/{endpoint:.*?}", controller="title", action="render_GET", conditions=dict(method=['GET']), requirements=dict(game=R"S..."))
+        self.map_get.connect(
-        self.map_post.connect("title_post", "/{game}/{version}/{endpoint:.*?}", controller="title", action="render_POST", conditions=dict(method=['POST']), requirements=dict(game=R"S..."))
+            "title_get",
+            "/{game}/{version}/{endpoint:.*?}",
+            controller="title",
+            action="render_GET",
+            conditions=dict(method=["GET"]),
+            requirements=dict(game=R"S..."),
+        )
+        self.map_post.connect(
+            "title_post",
+            "/{game}/{version}/{endpoint:.*?}",
+            controller="title",
+            action="render_POST",
+            conditions=dict(method=["POST"]),
+            requirements=dict(game=R"S..."),
+        )
 
-    def render_GET(self, request: Request) -> bytes:    
+    def render_GET(self, request: Request) -> bytes:
         test = self.map_get.match(request.uri.decode())
+        client_ip = Utils.get_ip_addr(request)
+
         if test is None:
-            self.logger.debug(f"Unknown GET endpoint {request.uri.decode()} from {request.getClientAddress().host} to port {request.getHost().port}")
+            self.logger.debug(
+                f"Unknown GET endpoint {request.uri.decode()} from {client_ip} to port {request.getHost().port}"
+            )
             request.setResponseCode(404)
             return b"Endpoint not found."
 
         return self.dispatch(test, request)
 
-    def render_POST(self, request: Request) -> bytes:    
+    def render_POST(self, request: Request) -> bytes:
         test = self.map_post.match(request.uri.decode())
+        client_ip = Utils.get_ip_addr(request)
+
         if test is None:
-            self.logger.debug(f"Unknown POST endpoint {request.uri.decode()} from {request.getClientAddress().host} to port {request.getHost().port}")
+            self.logger.debug(
+                f"Unknown POST endpoint {request.uri.decode()} from {client_ip} to port {request.getHost().port}"
+            )
             request.setResponseCode(404)
             return b"Endpoint not found."
-        
+
         return self.dispatch(test, request)
 
     def dispatch(self, matcher: Dict, request: Request) -> bytes:
         controller = getattr(self, matcher["controller"], None)
         if controller is None:
-            self.logger.error(f"Controller {matcher['controller']} not found via endpoint {request.uri.decode()}")
+            self.logger.error(
+                f"Controller {matcher['controller']} not found via endpoint {request.uri.decode()}"
+            )
             request.setResponseCode(404)
             return b"Endpoint not found."
-        
+
         handler = getattr(controller, matcher["action"], None)
         if handler is None:
-            self.logger.error(f"Action {matcher['action']} not found in controller {matcher['controller']} via endpoint {request.uri.decode()}")
+            self.logger.error(
+                f"Action {matcher['action']} not found in controller {matcher['controller']} via endpoint {request.uri.decode()}"
+            )
             request.setResponseCode(404)
             return b"Endpoint not found."
-        
+
         url_vars = matcher
         url_vars.pop("controller")
         url_vars.pop("action")
         ret = handler(request, url_vars)
-        
+
         if type(ret) == str:
             return ret.encode()
         elif type(ret) == bytes:
             return ret
         else:
             return b""
-        
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="ARTEMiS main entry point")
-    parser.add_argument("--config", "-c", type=str, default="config", help="Configuration folder")
+    parser.add_argument(
+        "--config", "-c", type=str, default="config", help="Configuration folder"
+    )
     args = parser.parse_args()
 
     if not path.exists(f"{args.config}/core.yaml"):
-        print(f"The config folder you specified ({args.config}) does not exist or does not contain core.yaml.\nDid you copy the example folder?")
+        print(
+            f"The config folder you specified ({args.config}) does not exist or does not contain core.yaml.\nDid you copy the example folder?"
+        )
         exit(1)
 
     cfg: CoreConfig = CoreConfig()
-    cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
+    if path.exists(f"{args.config}/core.yaml"):
+        cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
+
+    if not path.exists(cfg.server.log_dir):
+        mkdir(cfg.server.log_dir)
+
+    if not access(cfg.server.log_dir, W_OK):
+        print(
+            f"Log directory {cfg.server.log_dir} NOT writable, please check permissions"
+        )
+        exit(1)
 
     logger = logging.getLogger("core")
     log_fmt_str = "[%(asctime)s] Core | %(levelname)s | %(message)s"
-    log_fmt = logging.Formatter(log_fmt_str)        
+    log_fmt = logging.Formatter(log_fmt_str)
 
-    fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(cfg.server.log_dir, "core"), when="d", backupCount=10)
+    fileHandler = TimedRotatingFileHandler(
+        "{0}/{1}.log".format(cfg.server.log_dir, "core"), when="d", backupCount=10
+    )
     fileHandler.setFormatter(log_fmt)
-    
+
     consoleHandler = logging.StreamHandler()
     consoleHandler.setFormatter(log_fmt)
 
     logger.addHandler(fileHandler)
     logger.addHandler(consoleHandler)
-    
+
     log_lv = logging.DEBUG if cfg.server.is_develop else logging.INFO
     logger.setLevel(log_lv)
     coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
 
-    if not path.exists(cfg.server.log_dir):
-        mkdir(cfg.server.log_dir)
-    
-    if not access(cfg.server.log_dir, W_OK):
-        logger.error(f"Log directory {cfg.server.log_dir} NOT writable, please check permissions")
-        exit(1)
-
     if not cfg.aimedb.key:
         logger.error("!!AIMEDB KEY BLANK, SET KEY IN CORE.YAML!!")
         exit(1)
-    
-    logger.info(f"ARTEMiS starting in {'develop' if cfg.server.is_develop else 'production'} mode")
 
-    allnet_server_str = f"tcp:{cfg.allnet.port}:interface={cfg.server.listen_address}"    
+    logger.info(
+        f"ARTEMiS starting in {'develop' if cfg.server.is_develop else 'production'} mode"
+    )
+
+    allnet_server_str = f"tcp:{cfg.allnet.port}:interface={cfg.server.listen_address}"
     title_server_str = f"tcp:{cfg.title.port}:interface={cfg.server.listen_address}"
     adb_server_str = f"tcp:{cfg.aimedb.port}:interface={cfg.server.listen_address}"
-    frontend_server_str = f"tcp:{cfg.frontend.port}:interface={cfg.server.listen_address}"
+    frontend_server_str = (
+        f"tcp:{cfg.frontend.port}:interface={cfg.server.listen_address}"
+    )
 
     billing_server_str = f"tcp:{cfg.billing.port}:interface={cfg.server.listen_address}"
     if cfg.server.is_develop:
-        billing_server_str = f"ssl:{cfg.billing.port}:interface={cfg.server.listen_address}"\
+        billing_server_str = (
+            f"ssl:{cfg.billing.port}:interface={cfg.server.listen_address}"
             f":privateKey={cfg.billing.ssl_key}:certKey={cfg.billing.ssl_cert}"
-    
+        )
+
     dispatcher = HttpDispatcher(cfg, args.config)
 
-    endpoints.serverFromString(reactor, allnet_server_str).listen(server.Site(dispatcher))
+    endpoints.serverFromString(reactor, allnet_server_str).listen(
+        server.Site(dispatcher)
+    )
     endpoints.serverFromString(reactor, adb_server_str).listen(AimedbFactory(cfg))
 
     if cfg.frontend.enable:
-        endpoints.serverFromString(reactor, frontend_server_str).listen(server.Site(FrontendServlet(cfg, args.config)))
+        endpoints.serverFromString(reactor, frontend_server_str).listen(
+            server.Site(FrontendServlet(cfg, args.config))
+        )
 
     if cfg.billing.port > 0:
-        endpoints.serverFromString(reactor, billing_server_str).listen(server.Site(dispatcher))
+        endpoints.serverFromString(reactor, billing_server_str).listen(
-    
+            server.Site(dispatcher)
-    if cfg.title.port > 0:        
+        )
-        endpoints.serverFromString(reactor, title_server_str).listen(server.Site(dispatcher))
+
-    
+    if cfg.title.port > 0:
-    reactor.run() # type: ignore
+        endpoints.serverFromString(reactor, title_server_str).listen(
+            server.Site(dispatcher)
+        )
+
+    reactor.run()  # type: ignore

read.py

@@ -3,65 +3,73 @@ import argparse
 import re
 import os
 import yaml
-import importlib
+from os import path
-import logging, coloredlogs
+import logging
+import coloredlogs
 
 from logging.handlers import TimedRotatingFileHandler
 from typing import List, Optional
 
-from core import CoreConfig
+from core import CoreConfig, Utils
-from core.utils import Utils
 
-class BaseReader():
+
-    def __init__(self, config: CoreConfig, version: int, bin_dir: Optional[str], opt_dir: Optional[str], extra: Optional[str]) -> None:
+class BaseReader:
+    def __init__(
+        self,
+        config: CoreConfig,
+        version: int,
+        bin_dir: Optional[str],
+        opt_dir: Optional[str],
+        extra: Optional[str],
+    ) -> None:
         self.logger = logging.getLogger("reader")
         self.config = config
         self.bin_dir = bin_dir
         self.opt_dir = opt_dir
         self.version = version
         self.extra = extra
-    
+
-    
     def get_data_directories(self, directory: str) -> List[str]:
         ret: List[str] = []
 
         for root, dirs, files in os.walk(directory):
-                for dir in dirs:
+            for dir in dirs:
-                    if re.fullmatch("[A-Z0-9]{4,4}", dir) is not None:
+                if re.fullmatch("[A-Z0-9]{4,4}", dir) is not None:
-                        ret.append(f"{root}/{dir}")
+                    ret.append(f"{root}/{dir}")
-        
+
         return ret
 
+
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='Import Game Information')
+    parser = argparse.ArgumentParser(description="Import Game Information")
     parser.add_argument(
-        '--series',
+        "--series",
-        action='store',
+        action="store",
         type=str,
         required=True,
-        help='The game series we are importing.',
+        help="The game series we are importing.",
     )
     parser.add_argument(
-        '--version',
+        "--version",
-        dest='version',
+        dest="version",
-        action='store',
+        action="store",
         type=int,
         required=True,
-        help='The game version we are importing.',
+        help="The game version we are importing.",
     )
     parser.add_argument(
-        '--binfolder',
+        "--binfolder",
-        dest='bin',
+        dest="bin",
-        action='store',
+        action="store",
         type=str,
-        help='Folder containing A000 base data',
+        help="Folder containing A000 base data",
     )
     parser.add_argument(
-        '--optfolder',
+        "--optfolder",
-        dest='opt',
+        dest="opt",
-        action='store',
+        action="store",
         type=str,
-        help='Folder containing Option data folders',
+        help="Folder containing Option data folders",
     )
     parser.add_argument(
         "--config",
@@ -79,21 +87,24 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     config = CoreConfig()
-    config.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
+    if path.exists(f"{args.config}/core.yaml"):
+        config.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
 
     log_fmt_str = "[%(asctime)s] Reader | %(levelname)s | %(message)s"
     log_fmt = logging.Formatter(log_fmt_str)
     logger = logging.getLogger("reader")
 
-    fileHandler = TimedRotatingFileHandler("{0}/{1}.log".format(config.server.log_dir, "reader"), when="d", backupCount=10)
+    fileHandler = TimedRotatingFileHandler(
+        "{0}/{1}.log".format(config.server.log_dir, "reader"), when="d", backupCount=10
+    )
     fileHandler.setFormatter(log_fmt)
-    
+
     consoleHandler = logging.StreamHandler()
     consoleHandler.setFormatter(log_fmt)
 
     logger.addHandler(fileHandler)
     logger.addHandler(consoleHandler)
-    
+
     log_lv = logging.DEBUG if config.server.is_develop else logging.INFO
     logger.setLevel(log_lv)
     coloredlogs.install(level=log_lv, logger=logger, fmt=log_fmt_str)
@@ -101,8 +112,8 @@ if __name__ == "__main__":
     if args.series is None or args.version is None:
         logger.error("Game or version not specified")
         parser.print_help()
-        exit(1) 
+        exit(1)
-    
+
     if args.bin is None and args.opt is None:
         logger.error("Must specify either bin or opt directory")
         parser.print_help()
@@ -112,7 +123,7 @@ if __name__ == "__main__":
         bin_arg = args.bin[:-1]
     else:
         bin_arg = args.bin
-    
+
     if args.opt is not None and (args.opt.endswith("\\") or args.opt.endswith("/")):
         opt_arg = args.opt[:-1]
     else:
@@ -124,7 +135,8 @@ if __name__ == "__main__":
 
     for dir, mod in titles.items():
         if args.series in mod.game_codes:
-            handler = mod.reader(config, args.version, bin_arg, opt_arg, args.extra)
+            handler = mod.reader(config, args.version,
+                                 bin_arg, opt_arg, args.extra)
             handler.read()
-    
+
     logger.info("Done")

readme.md

@@ -15,6 +15,10 @@ Games listed below have been tested and confirmed working. Only game versions ol
 + Hatsune Miku Arcade
     + All versions
 
++ Card Maker
+    + 1.34.xx
+    + 1.35.xx
+
 + Ongeki
     + All versions up to Bright Memory
 
@@ -32,5 +36,8 @@ Games listed below have been tested and confirmed working. Only game versions ol
 ## Setup guides
 Follow the platform-specific guides for [windows](docs/INSTALL_WINDOWS.md) and [ubuntu](docs/INSTALL_UBUNTU.md) to setup and run the server.
 
+## Game specific information
+Read [Games specific info](docs/game_specific_info.md) for all supported games, importer settings, configuration option and database upgrades.
+
 ## Production guide
 See the [production guide](docs/prod.md) for running a production server.

titles/chuni/__init__.py

@@ -6,13 +6,5 @@ from titles.chuni.read import ChuniReader
 index = ChuniServlet
 database = ChuniData
 reader = ChuniReader
-
-use_default_title = True
-include_protocol = True
-title_secure = False
 game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW]
-trailing_slash = True
-use_default_host = False
-host = ""
-
 current_schema_version = 1

titles/chuni/air.py

@@ -5,12 +5,13 @@ from titles.chuni.base import ChuniBase
 from titles.chuni.const import ChuniConstants
 from titles.chuni.config import ChuniConfig
 
+
 class ChuniAir(ChuniBase):
     def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AIR
-    
+
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret =  super().handle_get_game_setting_api_request(data)
+        ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.10.00"
-        return ret
+        return ret

titles/chuni/airplus.py

@@ -5,12 +5,13 @@ from titles.chuni.base import ChuniBase
 from titles.chuni.const import ChuniConstants
 from titles.chuni.config import ChuniConfig
 
+
 class ChuniAirPlus(ChuniBase):
     def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AIR_PLUS
-    
+
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret =  super().handle_get_game_setting_api_request(data)
+        ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.15.00"
-        return ret
+        return ret

titles/chuni/amazon.py

@@ -7,12 +7,13 @@ from titles.chuni.base import ChuniBase
 from titles.chuni.const import ChuniConstants
 from titles.chuni.config import ChuniConfig
 
+
 class ChuniAmazon(ChuniBase):
     def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AMAZON
-    
+
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret =  super().handle_get_game_setting_api_request(data)
+        ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.30.00"
         return ret

titles/chuni/amazonplus.py

@@ -7,12 +7,13 @@ from titles.chuni.base import ChuniBase
 from titles.chuni.const import ChuniConstants
 from titles.chuni.config import ChuniConfig
 
+
 class ChuniAmazonPlus(ChuniBase):
     def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_AMAZON_PLUS
-    
+
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret =  super().handle_get_game_setting_api_request(data)
+        ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.35.00"
         return ret

titles/chuni/base.py

@@ -11,7 +11,8 @@ from titles.chuni.const import ChuniConstants
 from titles.chuni.database import ChuniData
 from titles.chuni.config import ChuniConfig
 
-class ChuniBase():
+
+class ChuniBase:
     def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
         self.core_cfg = core_cfg
         self.game_cfg = game_cfg
@@ -20,38 +21,48 @@ class ChuniBase():
         self.logger = logging.getLogger("chuni")
         self.game = ChuniConstants.GAME_CODE
         self.version = ChuniConstants.VER_CHUNITHM
-    
+
     def handle_game_login_api_request(self, data: Dict) -> Dict:
-        #self.data.base.log_event("chuni", "login", logging.INFO, {"version": self.version, "user": data["userId"]})
+        # self.data.base.log_event("chuni", "login", logging.INFO, {"version": self.version, "user": data["userId"]})
-        return { "returnCode": 1 }
+        return {"returnCode": 1}
-    
+
     def handle_game_logout_api_request(self, data: Dict) -> Dict:
-        #self.data.base.log_event("chuni", "logout", logging.INFO, {"version": self.version, "user": data["userId"]})
+        # self.data.base.log_event("chuni", "logout", logging.INFO, {"version": self.version, "user": data["userId"]})
-        return { "returnCode": 1 }
+        return {"returnCode": 1}
 
     def handle_get_game_charge_api_request(self, data: Dict) -> Dict:
         game_charge_list = self.data.static.get_enabled_charges(self.version)
+        
+        if game_charge_list is None or len(game_charge_list) == 0:
+            return {"length": 0, "gameChargeList": []}
 
         charges = []
-        for x in range(len(game_charge_list)):            
+        for x in range(len(game_charge_list)):
-            charges.append({
+            charges.append(
-                "orderId": x,
+                {
-                "chargeId": game_charge_list[x]["chargeId"],
+                    "orderId": x,
-                "price": 1,
+                    "chargeId": game_charge_list[x]["chargeId"],
-                "startDate": "2017-12-05 07:00:00.0", 
+                    "price": 1,
-                "endDate": "2099-12-31 00:00:00.0", 
+                    "startDate": "2017-12-05 07:00:00.0",
-                "salePrice": 1, 
+                    "endDate": "2099-12-31 00:00:00.0",
-                "saleStartDate": "2017-12-05 07:00:00.0", 
+                    "salePrice": 1,
-                "saleEndDate": "2099-12-31 00:00:00.0"
+                    "saleStartDate": "2017-12-05 07:00:00.0",
-            })
+                    "saleEndDate": "2099-12-31 00:00:00.0",
-        return {
+                }
-            "length": len(charges), 
+            )
-            "gameChargeList": charges
+        return {"length": len(charges), "gameChargeList": charges}
-        }
 
     def handle_get_game_event_api_request(self, data: Dict) -> Dict:
         game_events = self.data.static.get_enabled_events(self.version)
 
+        if game_events is None or len(game_events) == 0:
+            self.logger.warn("No enabled events, did you run the reader?")
+            return {
+                "type": data["type"],
+                "length": 0,
+                "gameEventList": [],
+            }
+
         event_list = []
         for evt_row in game_events:
             tmp = {}
@@ -62,26 +73,30 @@ class ChuniBase():
             event_list.append(tmp)
 
         return {
-            "type": data["type"], 
+            "type": data["type"],
-            "length": len(event_list), 
+            "length": len(event_list),
-            "gameEventList": event_list
+            "gameEventList": event_list,
         }
 
     def handle_get_game_idlist_api_request(self, data: Dict) -> Dict:
-        return { "type": data["type"], "length": 0, "gameIdlistList": [] }
+        return {"type": data["type"], "length": 0, "gameIdlistList": []}
 
     def handle_get_game_message_api_request(self, data: Dict) -> Dict:
-        return { "type": data["type"], "length": "0", "gameMessageList": [] }
+        return {"type": data["type"], "length": "0", "gameMessageList": []}
 
     def handle_get_game_ranking_api_request(self, data: Dict) -> Dict:
-        return { "type": data["type"], "gameRankingList": [] }
+        return {"type": data["type"], "gameRankingList": []}
 
     def handle_get_game_sale_api_request(self, data: Dict) -> Dict:
-        return { "type": data["type"], "length": 0, "gameSaleList": [] }
+        return {"type": data["type"], "length": 0, "gameSaleList": []}
 
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        reboot_start = datetime.strftime(datetime.now() - timedelta(hours=4), self.date_time_format)
+        reboot_start = datetime.strftime(
-        reboot_end = datetime.strftime(datetime.now() - timedelta(hours=3), self.date_time_format)
+            datetime.now() - timedelta(hours=4), self.date_time_format
+        )
+        reboot_end = datetime.strftime(
+            datetime.now() - timedelta(hours=3), self.date_time_format
+        )
         return {
             "gameSetting": {
                 "dataVersion": "1.00.00",
@@ -94,15 +109,17 @@ class ChuniBase():
                 "maxCountItem": 300,
                 "maxCountMusic": 300,
             },
-                "isDumpUpload": "false",
+            "isDumpUpload": "false",
-                "isAou": "false",
+            "isAou": "false",
         }
 
     def handle_get_user_activity_api_request(self, data: Dict) -> Dict:
-        user_activity_list = self.data.profile.get_profile_activity(data["userId"], data["kind"])
+        user_activity_list = self.data.profile.get_profile_activity(
-        
+            data["userId"], data["kind"]
+        )
+
         activity_list = []
-        
+
         for activity in user_activity_list:
             tmp = activity._asdict()
             tmp.pop("user")
@@ -111,15 +128,16 @@ class ChuniBase():
             activity_list.append(tmp)
 
         return {
-          "userId": data["userId"], 
+            "userId": data["userId"],
-          "length": len(activity_list),
+            "length": len(activity_list),
-          "kind": data["kind"],
+            "kind": data["kind"],
-          "userActivityList": activity_list
+            "userActivityList": activity_list,
         }
 
     def handle_get_user_character_api_request(self, data: Dict) -> Dict:
         characters = self.data.item.get_characters(data["userId"])
-        if characters is None: return {}
+        if characters is None:
+            return {}
         next_idx = -1
 
         characterList = []
@@ -131,15 +149,17 @@ class ChuniBase():
 
             if len(characterList) >= int(data["maxCount"]):
                 break
-        
+
-        if len(characterList) >= int(data["maxCount"]) and len(characters) > int(data["maxCount"]) + int(data["nextIndex"]):
+        if len(characterList) >= int(data["maxCount"]) and len(characters) > int(
+            data["maxCount"]
+        ) + int(data["nextIndex"]):
             next_idx = int(data["maxCount"]) + int(data["nextIndex"]) + 1
-        
+
         return {
-            "userId": data["userId"], 
+            "userId": data["userId"],
             "length": len(characterList),
-            "nextIndex": next_idx, 
+            "nextIndex": next_idx,
-            "userCharacterList": characterList
+            "userCharacterList": characterList,
         }
 
     def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
@@ -153,21 +173,21 @@ class ChuniBase():
             charge_list.append(tmp)
 
         return {
-            "userId": data["userId"], 
+            "userId": data["userId"],
             "length": len(charge_list),
-            "userChargeList": charge_list
+            "userChargeList": charge_list,
         }
 
     def handle_get_user_course_api_request(self, data: Dict) -> Dict:
         user_course_list = self.data.score.get_courses(data["userId"])
-        if user_course_list is None: 
+        if user_course_list is None:
             return {
-                "userId": data["userId"], 
+                "userId": data["userId"],
                 "length": 0,
-                "nextIndex": -1, 
+                "nextIndex": -1,
-                "userCourseList": []
+                "userCourseList": [],
             }
-        
+
         course_list = []
         next_idx = int(data["nextIndex"])
         max_ct = int(data["maxCount"])
@@ -180,51 +200,48 @@ class ChuniBase():
 
             if len(user_course_list) >= max_ct:
                 break
-        
+
         if len(user_course_list) >= max_ct:
             next_idx = next_idx + max_ct
         else:
             next_idx = -1
-                
+
         return {
-            "userId": data["userId"], 
+            "userId": data["userId"],
             "length": len(course_list),
-            "nextIndex": next_idx, 
+            "nextIndex": next_idx,
-            "userCourseList": course_list
+            "userCourseList": course_list,
         }
 
     def handle_get_user_data_api_request(self, data: Dict) -> Dict:
         p = self.data.profile.get_profile_data(data["userId"], self.version)
-        if p is None: return {}
+        if p is None:
+            return {}
 
         profile = p._asdict()
         profile.pop("id")
         profile.pop("user")
         profile.pop("version")
 
-        return {
+        return {"userId": data["userId"], "userData": profile}
-            "userId": data["userId"], 
-            "userData": profile
-        }
 
     def handle_get_user_data_ex_api_request(self, data: Dict) -> Dict:
         p = self.data.profile.get_profile_data_ex(data["userId"], self.version)
-        if p is None: return {}
+        if p is None:
+            return {}
 
         profile = p._asdict()
         profile.pop("id")
         profile.pop("user")
         profile.pop("version")
 
-        return {
+        return {"userId": data["userId"], "userDataEx": profile}
-            "userId": data["userId"], 
-            "userDataEx": profile
-        }
 
     def handle_get_user_duel_api_request(self, data: Dict) -> Dict:
         user_duel_list = self.data.item.get_duels(data["userId"])
-        if user_duel_list is None: return {}
+        if user_duel_list is None:
-        
+            return {}
+
         duel_list = []
         for duel in user_duel_list:
             tmp = duel._asdict()
@@ -233,18 +250,18 @@ class ChuniBase():
             duel_list.append(tmp)
 
         return {
-            "userId": data["userId"], 
+            "userId": data["userId"],
             "length": len(duel_list),
-            "userDuelList": duel_list
+            "userDuelList": duel_list,
         }
 
     def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
         return {
-            "userId": data["userId"], 
+            "userId": data["userId"],
             "length": 0,
-            "kind": data["kind"], 
+            "kind": data["kind"],
-            "nextIndex": -1, 
+            "nextIndex": -1,
-            "userFavoriteItemList": []
+            "userFavoriteItemList": [],
         }
 
     def handle_get_user_favorite_music_api_request(self, data: Dict) -> Dict:
@@ -252,22 +269,23 @@ class ChuniBase():
         This is handled via the webui, which we don't have right now
         """
 
-        return {
+        return {"userId": data["userId"], "length": 0, "userFavoriteMusicList": []}
-            "userId": data["userId"], 
-            "length": 0,
-            "userFavoriteMusicList": []
-        }
 
     def handle_get_user_item_api_request(self, data: Dict) -> Dict:
         kind = int(int(data["nextIndex"]) / 10000000000)
         next_idx = int(int(data["nextIndex"]) % 10000000000)
         user_item_list = self.data.item.get_items(data["userId"], kind)
 
-        if user_item_list is None or len(user_item_list) == 0: 
+        if user_item_list is None or len(user_item_list) == 0:
-            return {"userId": data["userId"], "nextIndex": -1, "itemKind": kind, "userItemList": []}
+            return {
+                "userId": data["userId"],
+                "nextIndex": -1,
+                "itemKind": kind,
+                "userItemList": [],
+            }
 
         items: list[Dict[str, Any]] = []
-        for i in range(next_idx, len(user_item_list)):            
+        for i in range(next_idx, len(user_item_list)):
             tmp = user_item_list[i]._asdict()
             tmp.pop("user")
             tmp.pop("id")
@@ -277,38 +295,47 @@ class ChuniBase():
 
         xout = kind * 10000000000 + next_idx + len(items)
 
-        if len(items) < int(data["maxCount"]): nextIndex = 0
+        if len(items) < int(data["maxCount"]):
-        else: nextIndex = xout
+            nextIndex = 0
+        else:
+            nextIndex = xout
 
-        return {"userId": data["userId"], "nextIndex": nextIndex, "itemKind": kind, "length": len(items), "userItemList": items}
+        return {
+            "userId": data["userId"],
+            "nextIndex": nextIndex,
+            "itemKind": kind,
+            "length": len(items),
+            "userItemList": items,
+        }
 
     def handle_get_user_login_bonus_api_request(self, data: Dict) -> Dict:
         """
         Unsure how to get this to trigger...
         """
         return {
-            "userId": data["userId"], 
+            "userId": data["userId"],
             "length": 2,
             "userLoginBonusList": [
                 {
-                "presetId": '10',
+                    "presetId": "10",
-                "bonusCount": '0',
+                    "bonusCount": "0",
-                "lastUpdateDate": "1970-01-01 09:00:00",
+                    "lastUpdateDate": "1970-01-01 09:00:00",
-                "isWatched": "true"
+                    "isWatched": "true",
                 },
                 {
-                "presetId": '20',
+                    "presetId": "20",
-                "bonusCount": '0',
+                    "bonusCount": "0",
-                "lastUpdateDate": "1970-01-01 09:00:00",
+                    "lastUpdateDate": "1970-01-01 09:00:00",
-                "isWatched": "true"
+                    "isWatched": "true",
                 },
-            ]
+            ],
         }
 
     def handle_get_user_map_api_request(self, data: Dict) -> Dict:
         user_map_list = self.data.item.get_maps(data["userId"])
-        if user_map_list is None: return {}
+        if user_map_list is None:
-        
+            return {}
+
         map_list = []
         for map in user_map_list:
             tmp = map._asdict()
@@ -317,19 +344,19 @@ class ChuniBase():
             map_list.append(tmp)
 
         return {
-            "userId": data["userId"], 
+            "userId": data["userId"],
             "length": len(map_list),
-            "userMapList": map_list
+            "userMapList": map_list,
         }
 
     def handle_get_user_music_api_request(self, data: Dict) -> Dict:
         music_detail = self.data.score.get_scores(data["userId"])
-        if music_detail is None: 
+        if music_detail is None:
             return {
-                "userId": data["userId"], 
+                "userId": data["userId"],
-                "length": 0, 
+                "length": 0,
                 "nextIndex": -1,
-                "userMusicList": [] #240
+                "userMusicList": [],  # 240
             }
         song_list = []
         next_idx = int(data["nextIndex"])
@@ -340,66 +367,60 @@ class ChuniBase():
             tmp = music_detail[x]._asdict()
             tmp.pop("user")
             tmp.pop("id")
-            
+
             for song in song_list:
                 if song["userMusicDetailList"][0]["musicId"] == tmp["musicId"]:
                     found = True
                     song["userMusicDetailList"].append(tmp)
                     song["length"] = len(song["userMusicDetailList"])
-            
+
             if not found:
-                song_list.append({
+                song_list.append({"length": 1, "userMusicDetailList": [tmp]})
-                    "length": 1,
+
-                    "userMusicDetailList": [tmp]
-                })
-            
             if len(song_list) >= max_ct:
                 break
-        
+
         if len(song_list) >= max_ct:
             next_idx += max_ct
         else:
             next_idx = 0
 
         return {
-            "userId": data["userId"], 
+            "userId": data["userId"],
-            "length": len(song_list), 
+            "length": len(song_list),
             "nextIndex": next_idx,
-            "userMusicList": song_list #240
+            "userMusicList": song_list,  # 240
         }
 
     def handle_get_user_option_api_request(self, data: Dict) -> Dict:
         p = self.data.profile.get_profile_option(data["userId"])
-        
+
         option = p._asdict()
         option.pop("id")
         option.pop("user")
 
-        return {
+        return {"userId": data["userId"], "userGameOption": option}
-            "userId": data["userId"], 
-            "userGameOption": option
-        }
 
     def handle_get_user_option_ex_api_request(self, data: Dict) -> Dict:
         p = self.data.profile.get_profile_option_ex(data["userId"])
-        
+
         option = p._asdict()
         option.pop("id")
         option.pop("user")
 
-        return {
+        return {"userId": data["userId"], "userGameOptionEx": option}
-            "userId": data["userId"], 
-            "userGameOptionEx": option
-        }
 
     def read_wtf8(self, src):
         return bytes([ord(c) for c in src]).decode("utf-8")
 
     def handle_get_user_preview_api_request(self, data: Dict) -> Dict:
         profile = self.data.profile.get_profile_preview(data["userId"], self.version)
-        if profile is None: return None
+        if profile is None:
-        profile_character = self.data.item.get_character(data["userId"], profile["characterId"])
+            return None
-        
+        profile_character = self.data.item.get_character(
+            data["userId"], profile["characterId"]
+        )
+
         if profile_character is None:
             chara = {}
         else:
@@ -408,8 +429,8 @@ class ChuniBase():
             chara.pop("user")
 
         return {
-            "userId": data["userId"],    
+            "userId": data["userId"],
-            # Current Login State            
+            # Current Login State
             "isLogin": False,
             "lastLoginDate": profile["lastPlayDate"],
             # User Profile
@@ -421,14 +442,14 @@ class ChuniBase():
             "lastGameId": profile["lastGameId"],
             "lastRomVersion": profile["lastRomVersion"],
             "lastDataVersion": profile["lastDataVersion"],
-            "lastPlayDate": profile["lastPlayDate"],            
+            "lastPlayDate": profile["lastPlayDate"],
-            "trophyId": profile["trophyId"],  
+            "trophyId": profile["trophyId"],
             "nameplateId": profile["nameplateId"],
             # Current Selected Character
             "userCharacter": chara,
             # User Game Options
-            "playerLevel": profile["playerLevel"], 
+            "playerLevel": profile["playerLevel"],
-            "rating": profile["rating"], 
+            "rating": profile["rating"],
             "headphone": profile["headphone"],
             "chargeState": "1",
             "userNameEx": profile["userName"],
@@ -436,7 +457,7 @@ class ChuniBase():
 
     def handle_get_user_recent_rating_api_request(self, data: Dict) -> Dict:
         recet_rating_list = self.data.profile.get_profile_recent_rating(data["userId"])
-        if recet_rating_list is None: 
+        if recet_rating_list is None:
             return {
                 "userId": data["userId"],
                 "length": 0,
@@ -459,11 +480,8 @@ class ChuniBase():
 
     def handle_get_user_team_api_request(self, data: Dict) -> Dict:
         # TODO: Team
-        return {
+        return {"userId": data["userId"], "teamId": 0}
-            "userId": data["userId"],
+
-            "teamId": 0
-        }
-    
     def handle_get_team_course_setting_api_request(self, data: Dict) -> Dict:
         return {
             "userId": data["userId"],
@@ -486,19 +504,30 @@ class ChuniBase():
 
         if "userData" in upsert:
             try:
-                upsert["userData"][0]["userName"] = self.read_wtf8(upsert["userData"][0]["userName"])
+                upsert["userData"][0]["userName"] = self.read_wtf8(
-            except: pass
+                    upsert["userData"][0]["userName"]
+                )
+            except:
+                pass
 
-            self.data.profile.put_profile_data(user_id, self.version, upsert["userData"][0])
+            self.data.profile.put_profile_data(
+                user_id, self.version, upsert["userData"][0]
+            )
         if "userDataEx" in upsert:
-            self.data.profile.put_profile_data_ex(user_id, self.version, upsert["userDataEx"][0])
+            self.data.profile.put_profile_data_ex(
+                user_id, self.version, upsert["userDataEx"][0]
+            )
         if "userGameOption" in upsert:
             self.data.profile.put_profile_option(user_id, upsert["userGameOption"][0])
         if "userGameOptionEx" in upsert:
-            self.data.profile.put_profile_option_ex(user_id, upsert["userGameOptionEx"][0])
+            self.data.profile.put_profile_option_ex(
+                user_id, upsert["userGameOptionEx"][0]
+            )
         if "userRecentRatingList" in upsert:
-            self.data.profile.put_profile_recent_rating(user_id, upsert["userRecentRatingList"])
+            self.data.profile.put_profile_recent_rating(
-        
+                user_id, upsert["userRecentRatingList"]
+            )
+
         if "userCharacterList" in upsert:
             for character in upsert["userCharacterList"]:
                 self.data.item.put_character(user_id, character)
@@ -514,7 +543,7 @@ class ChuniBase():
         if "userDuelList" in upsert:
             for duel in upsert["userDuelList"]:
                 self.data.item.put_duel(user_id, duel)
-        
+
         if "userItemList" in upsert:
             for item in upsert["userItemList"]:
                 self.data.item.put_item(user_id, item)
@@ -522,23 +551,23 @@ class ChuniBase():
         if "userActivityList" in upsert:
             for activity in upsert["userActivityList"]:
                 self.data.profile.put_profile_activity(user_id, activity)
-        
+
         if "userChargeList" in upsert:
             for charge in upsert["userChargeList"]:
                 self.data.profile.put_profile_charge(user_id, charge)
-        
+
         if "userMusicDetailList" in upsert:
             for song in upsert["userMusicDetailList"]:
                 self.data.score.put_score(user_id, song)
-        
+
         if "userPlaylogList" in upsert:
             for playlog in upsert["userPlaylogList"]:
                 self.data.score.put_playlog(user_id, playlog)
-        
+
         if "userTeamPoint" in upsert:
             # TODO: team stuff
             pass
-        
+
         if "userMapAreaList" in upsert:
             for map_area in upsert["userMapAreaList"]:
                 self.data.item.put_map_area(user_id, map_area)
@@ -551,22 +580,30 @@ class ChuniBase():
             for emoney in upsert["userEmoneyList"]:
                 self.data.profile.put_profile_emoney(user_id, emoney)
 
-        return { "returnCode": "1" }
+        return {"returnCode": "1"}
 
     def handle_upsert_user_chargelog_api_request(self, data: Dict) -> Dict:
-        return { "returnCode": "1" }
+        return {"returnCode": "1"}
 
     def handle_upsert_client_bookkeeping_api_request(self, data: Dict) -> Dict:
-        return { "returnCode": "1" }
+        return {"returnCode": "1"}
 
     def handle_upsert_client_develop_api_request(self, data: Dict) -> Dict:
-        return { "returnCode": "1" }
+        return {"returnCode": "1"}
 
     def handle_upsert_client_error_api_request(self, data: Dict) -> Dict:
-        return { "returnCode": "1" }
+        return {"returnCode": "1"}
 
     def handle_upsert_client_setting_api_request(self, data: Dict) -> Dict:
-        return { "returnCode": "1" }
+        return {"returnCode": "1"}
 
     def handle_upsert_client_testmode_api_request(self, data: Dict) -> Dict:
-        return { "returnCode": "1" }
+        return {"returnCode": "1"}
+
+    def handle_get_user_net_battle_data_api_request(self, data: Dict) -> Dict:
+        return {
+            "userId": data["userId"],
+            "userNetBattleData": {
+                "recentNBSelectMusicList": []
+            }
+        }

titles/chuni/config.py

@@ -1,36 +1,49 @@
 from core.config import CoreConfig
 from typing import Dict
 
-class ChuniServerConfig():
+
+class ChuniServerConfig:
     def __init__(self, parent_config: "ChuniConfig") -> None:
         self.__config = parent_config
-    
+
     @property
     def enable(self) -> bool:
-        return CoreConfig.get_config_field(self.__config, 'chuni', 'server', 'enable', default=True)
+        return CoreConfig.get_config_field(
-    
+            self.__config, "chuni", "server", "enable", default=True
+        )
+
     @property
     def loglevel(self) -> int:
-        return CoreConfig.str_to_loglevel(CoreConfig.get_config_field(self.__config, 'chuni', 'server', 'loglevel', default="info"))
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "chuni", "server", "loglevel", default="info"
+            )
+        )
 
-class ChuniCryptoConfig():
+
+class ChuniCryptoConfig:
     def __init__(self, parent_config: "ChuniConfig") -> None:
         self.__config = parent_config
-    
+
     @property
     def keys(self) -> Dict:
         """
         in the form of:
-        internal_version: [key, iv]        
+        internal_version: [key, iv]
         all values are hex strings
         """
-        return CoreConfig.get_config_field(self.__config, 'chuni', 'crypto', 'keys', default={})
+        return CoreConfig.get_config_field(
+            self.__config, "chuni", "crypto", "keys", default={}
+        )
 
     @property
     def encrypted_only(self) -> bool:
-        return CoreConfig.get_config_field(self.__config, 'chuni', 'crypto', 'encrypted_only', default=False)
+        return CoreConfig.get_config_field(
+            self.__config, "chuni", "crypto", "encrypted_only", default=False
+        )
+
 
 class ChuniConfig(dict):
     def __init__(self) -> None:
         self.server = ChuniServerConfig(self)
-        self.crypto = ChuniCryptoConfig(self)
+        self.crypto = ChuniCryptoConfig(self)

titles/chuni/const.py

@@ -1,12 +1,14 @@
-class ChuniConstants():
+class ChuniConstants:
     GAME_CODE = "SDBT"
     GAME_CODE_NEW = "SDHD"
 
+    CONFIG_NAME = "chuni.yaml"
+
     VER_CHUNITHM = 0
     VER_CHUNITHM_PLUS = 1
     VER_CHUNITHM_AIR = 2
     VER_CHUNITHM_AIR_PLUS = 3
-    VER_CHUNITHM_STAR = 4 
+    VER_CHUNITHM_STAR = 4
     VER_CHUNITHM_STAR_PLUS = 5
     VER_CHUNITHM_AMAZON = 6
     VER_CHUNITHM_AMAZON_PLUS = 7
@@ -16,8 +18,21 @@ class ChuniConstants():
     VER_CHUNITHM_NEW = 11
     VER_CHUNITHM_NEW_PLUS = 12
 
-    VERSION_NAMES = ["Chunithm", "Chunithm+", "Chunithm Air", "Chunithm Air+", "Chunithm Star", "Chunithm Star+", "Chunithm Amazon",
+    VERSION_NAMES = [
-    "Chunithm Amazon+", "Chunithm Crystal", "Chunithm Crystal+", "Chunithm Paradise", "Chunithm New!!", "Chunithm New!!+"]
+        "Chunithm",
+        "Chunithm+",
+        "Chunithm Air",
+        "Chunithm Air+",
+        "Chunithm Star",
+        "Chunithm Star+",
+        "Chunithm Amazon",
+        "Chunithm Amazon+",
+        "Chunithm Crystal",
+        "Chunithm Crystal+",
+        "Chunithm Paradise",
+        "Chunithm New!!",
+        "Chunithm New!!+",
+    ]
 
     @classmethod
     def game_ver_to_string(cls, ver: int):

titles/chuni/crystal.py

@@ -7,12 +7,13 @@ from titles.chuni.base import ChuniBase
 from titles.chuni.const import ChuniConstants
 from titles.chuni.config import ChuniConfig
 
+
 class ChuniCrystal(ChuniBase):
     def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_CRYSTAL
-    
+
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret =  super().handle_get_game_setting_api_request(data)
+        ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.40.00"
         return ret

titles/chuni/crystalplus.py

@@ -7,12 +7,13 @@ from titles.chuni.base import ChuniBase
 from titles.chuni.const import ChuniConstants
 from titles.chuni.config import ChuniConfig
 
+
 class ChuniCrystalPlus(ChuniBase):
     def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
         super().__init__(core_cfg, game_cfg)
         self.version = ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS
-    
+
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
-        ret =  super().handle_get_game_setting_api_request(data)
+        ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.45.00"
         return ret

titles/chuni/database.py

@@ -2,6 +2,7 @@ from core.data import Data
 from core.config import CoreConfig
 from titles.chuni.schema import *
 
+
 class ChuniData(Data):
     def __init__(self, cfg: CoreConfig) -> None:
         super().__init__(cfg)
@@ -9,4 +10,4 @@ class ChuniData(Data):
         self.item = ChuniItemData(cfg, self.session)
         self.profile = ChuniProfileData(cfg, self.session)
         self.score = ChuniScoreData(cfg, self.session)
-        self.static = ChuniStaticData(cfg, self.session)
+        self.static = ChuniStaticData(cfg, self.session)