ongeki: card maker config added, small fixes, improved credits

- Changed the credits config to the default 370 instead of 360
- Added `start_date` to the events to show new events
- Fixed Card Maker Gachas to only allow "Select Gacha" once
- Fixed the `get_profile_rating_log` database query

.gitignore

@@ -158,5 +158,6 @@ cert/*
 !cert/server.pem
 config/*
 deliver/*
+*.gz
 
 dbdump-*.json

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.9.15-slim-bullseye
+
+RUN apt update && apt install default-libmysqlclient-dev build-essential libtk nodejs npm -y
+
+WORKDIR /app
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+RUN npm i -g nodemon
+
+COPY entrypoint.sh entrypoint.sh
+RUN chmod +x entrypoint.sh
+
+COPY index.py index.py
+COPY dbutils.py dbutils.py
+ADD core core
+ADD titles titles
+ADD config config
+ADD log log
+ADD cert cert
+
+ENTRYPOINT [ "/app/entrypoint.sh" ]

changelog.md

@@ -0,0 +1,49 @@
+# Changelog
+Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to.
+
+## 2023042300
+### Wacca
++ Time free now works properly
++ Fix reverse gate mission causing a fatal error
++ Other misc. fixes
++ Latest DB: 5
+
+### Pokken
++ Added preliminary support
+    + Nothing saves currently, but the game will boot and function properly.
+
+### Initial D Zero
++ Added preliminary support
+    + Nothing saves currently, but the game will boot and function for the most part.
+
+### Mai2
++ Added support for Festival
++ Lasted DB Version: 4
+
+### Ongeki
++ Misc fixes
++ Lasted DB Version: 4
+
+### Diva
++ Misc fixes
++ Lasted DB Version: 4
+
+### Chuni
++ Fix network encryption
++ Add `handle_remove_token_api_request` for event mode
+
+### Allnet
++ Added download order support
+    + It is up to the sysop to provide the INI file, and host the files.
+    + ONLY for use with cabs. It's not checked currently, which it's why it's default disabled
+    + YMMV, use at your own risk
++ When running develop mode, games that are not recognised will still be able to authenticate.
+
+### Database
++ Add autoupgrade command
+    + Invoke to automatically upgrade all schemas to their latest versions
+
++ `version` arg no longer required, leave it blank to update the game schema to latest if it isn't already
+
+### Misc
++ Update example nginx config file

core/allnet.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Any, Optional, Tuple
+from typing import Dict, List, Any, Optional, Tuple, Union
 import logging, coloredlogs
 from logging.handlers import TimedRotatingFileHandler
 from twisted.web.http import Request
@@ -10,6 +10,8 @@ from Crypto.PublicKey import RSA
 from Crypto.Hash import SHA
 from Crypto.Signature import PKCS1_v1_5
 from time import strptime
+from os import path
+import urllib.parse
 
 from core.config import CoreConfig
 from core.utils import Utils
@@ -55,7 +57,7 @@ class AllnetServlet:
             self.logger.error("No games detected!")
 
         for _, mod in plugins.items():
-            if hasattr(mod.index, "get_allnet_info"):
+            if hasattr(mod, "index") and hasattr(mod.index, "get_allnet_info"):
                 for code in mod.game_codes:
                     enabled, uri, host = mod.index.get_allnet_info(
                         code, self.config, self.config_folder
@@ -78,7 +80,7 @@ class AllnetServlet:
             req = AllnetPowerOnRequest(req_dict[0])
             # Validate the request. Currently we only validate the fields we plan on using
 
-            if not req.game_id or not req.ver or not req.serial or not req.ip:
+            if not req.game_id or not req.ver or not req.serial or not req.ip or not req.firm_ver or not req.boot_ver:
                 raise AllnetRequestException(
                     f"Bad auth request params from {request_ip} - {vars(req)}"
                 )
@@ -88,10 +90,12 @@ class AllnetServlet:
                 self.logger.error(e)
             return b""
 
-        if req.format_ver == "3":
+        if req.format_ver == 3:
             resp = AllnetPowerOnResponse3(req.token)
-        else:
+        elif req.format_ver == 2:
             resp = AllnetPowerOnResponse2()
+        else:
+            resp = AllnetPowerOnResponse()
 
         self.logger.debug(f"Allnet request: {vars(req)}")        
         if req.game_id not in self.uri_registry:
@@ -102,14 +106,22 @@ class AllnetServlet:
                 )
                 self.logger.warn(msg)
 
-                resp.stat = 0
+                resp.stat = -1
-                return self.dict_to_http_form_string([vars(resp)])
+                resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
 
             else:
-                self.logger.info(f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}")
+                self.logger.info(
+                    f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}"
+                )
                 resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
                 resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
-                return self.dict_to_http_form_string([vars(resp)])
+                
+                resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+                resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
+                
+                self.logger.debug(f"Allnet response: {resp_str}")
+                return (resp_str + "\n").encode("utf-8")
 
         resp.uri, resp.host = self.uri_registry[req.game_id]
 
@@ -121,8 +133,9 @@ class AllnetServlet:
             )
             self.logger.warn(msg)
 
-            resp.stat = 0
+            resp.stat = -2
-            return self.dict_to_http_form_string([vars(resp)])
+            resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+            return (urllib.parse.unquote(urllib.parse.urlencode(resp_dict)) + "\n").encode("utf-8")
 
         if machine is not None:
             arcade = self.data.arcade.get_arcade(machine["arcade"])
@@ -164,9 +177,13 @@ class AllnetServlet:
         msg = f"{req.serial} authenticated from {request_ip}: {req.game_id} v{req.ver}"
         self.data.base.log_event("allnet", "ALLNET_AUTH_SUCCESS", logging.INFO, msg)
         self.logger.info(msg)
-        self.logger.debug(f"Allnet response: {vars(resp)}")
 
-        return self.dict_to_http_form_string([vars(resp)]).encode("utf-8")
+        resp_dict = {k: v for k, v in vars(resp).items() if v is not None}
+        resp_str = urllib.parse.unquote(urllib.parse.urlencode(resp_dict))
+        self.logger.debug(f"Allnet response: {resp_dict}")        
+        resp_str += "\n"
+
+        return resp_str.encode("utf-8")
 
     def handle_dlorder(self, request: Request, _: Dict):
         request_ip = Utils.get_ip_addr(request)
@@ -188,14 +205,75 @@ class AllnetServlet:
                 self.logger.error(e)
             return b""
 
-        self.logger.info(f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}")
+        self.logger.info(
-        resp = AllnetDownloadOrderResponse()
+            f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}"
+        )
+        resp = AllnetDownloadOrderResponse(serial=req.serial)
 
-        if not self.config.allnet.allow_online_updates:
+        if (
-            return self.dict_to_http_form_string([vars(resp)])
+            not self.config.allnet.allow_online_updates
+            or not self.config.allnet.update_cfg_folder
+        ):
+            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
 
-        else:  # TODO: Actual dlorder response
+        else:  # TODO: Keychip check
-            return self.dict_to_http_form_string([vars(resp)])
+            if path.exists(
+                f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
+            ):
+                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
+
+            if path.exists(
+                f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
+            ):
+                resp.uri += f"|http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
+
+            self.logger.debug(f"Sending download uri {resp.uri}")
+            self.data.base.log_event("allnet", "DLORDER_REQ_SUCCESS", logging.INFO, f"{Utils.get_ip_addr(request)} requested DL Order for {req.serial} {req.game_id} v{req.ver}")
+
+            return urllib.parse.unquote(urllib.parse.urlencode(vars(resp))) + "\n"
+
+    def handle_dlorder_ini(self, request: Request, match: Dict) -> bytes:
+        if "file" not in match:
+            return b""
+
+        req_file = match["file"].replace("%0A", "")
+
+        if path.exists(f"{self.config.allnet.update_cfg_folder}/{req_file}"):
+            self.logger.info(f"Request for DL INI file {req_file} from {Utils.get_ip_addr(request)} successful")
+            self.data.base.log_event("allnet", "DLORDER_INI_SENT", logging.INFO, f"{Utils.get_ip_addr(request)} successfully recieved {req_file}")
+            return open(
+                f"{self.config.allnet.update_cfg_folder}/{req_file}", "rb"
+            ).read()
+
+        self.logger.info(f"DL INI File {req_file} not found")
+        return b""
+
+    def handle_dlorder_report(self, request: Request, match: Dict) -> bytes:
+        self.logger.info(
+            f"DLI Report from {Utils.get_ip_addr(request)}: {request.content.getvalue()}"
+        )
+        return b""
+
+    def handle_loaderstaterecorder(self, request: Request, match: Dict) -> bytes:
+        req_data = request.content.getvalue()
+        sections = req_data.decode("utf-8").split("\r\n")
+        
+        req_dict = dict(urllib.parse.parse_qsl(sections[0]))
+
+        serial: Union[str, None] = req_dict.get("serial", None)
+        num_files_to_dl: Union[str, None] = req_dict.get("nb_ftd", None)
+        num_files_dld: Union[str, None] = req_dict.get("nb_dld", None)
+        dl_state: Union[str, None] = req_dict.get("dld_st", None)
+        ip = Utils.get_ip_addr(request)
+
+        if serial is None or num_files_dld is None or num_files_to_dl is None or dl_state is None:
+            return "NG".encode()
+
+        self.logger.info(f"LoaderStateRecorder Request from {ip} {serial}: {num_files_dld}/{num_files_to_dl} Files download (State: {dl_state})")
+        return "OK".encode()
+    
+    def handle_alive(self, request: Request, match: Dict) -> bytes:
+        return "OK".encode()
 
     def handle_billing_request(self, request: Request, _: Dict):
         req_dict = self.billing_req_to_dict(request.content.getvalue())
@@ -210,6 +288,7 @@ class AllnetServlet:
         signer = PKCS1_v1_5.new(rsa)
         digest = SHA.new()
 
+        try:
             kc_playlimit = int(req_dict[0]["playlimit"])
             kc_nearfull = int(req_dict[0]["nearfull"])
             kc_billigtype = int(req_dict[0]["billingtype"])
@@ -219,6 +298,9 @@ class AllnetServlet:
             kc_date = strptime(req_dict[0]["date"], "%Y%m%d%H%M%S")
             kc_serial_bytes = kc_serial.encode()
         
+        except KeyError as e:
+            return f"result=5&linelimit=&message={e} field is missing".encode()
+
         machine = self.data.arcade.get_machine(kc_serial)
         if machine is None and not self.config.server.allow_unregistered_serials:
             msg = f"Unrecognised serial {kc_serial} attempted billing checkin from {request_ip} for game {kc_game}."
@@ -256,7 +338,7 @@ class AllnetServlet:
 
         resp = BillingResponse(playlimit, playlimit_sig, nearfull, nearfull_sig)
 
-        resp_str = self.dict_to_http_form_string([vars(resp)], True)
+        resp_str = self.dict_to_http_form_string([vars(resp)])
         if resp_str is None:
             self.logger.error(f"Failed to parse response {vars(resp)}")
 
@@ -267,21 +349,6 @@ class AllnetServlet:
         self.logger.info(f"Ping from {Utils.get_ip_addr(request)}")
         return b"naomi ok"
 
-    def kvp_to_dict(self, kvp: List[str]) -> List[Dict[str, Any]]:
-        ret: List[Dict[str, Any]] = []
-        for x in kvp:
-            items = x.split("&")
-            tmp = {}
-
-            for item in items:
-                kvp = item.split("=")
-                if len(kvp) == 2:
-                    tmp[kvp[0]] = kvp[1]
-
-            ret.append(tmp)
-
-        return ret
-
     def billing_req_to_dict(self, data: bytes):
         """
         Parses an billing request string into a python dictionary
@@ -291,7 +358,10 @@ class AllnetServlet:
             unzipped = decomp.decompress(data)
             sections = unzipped.decode("ascii").split("\r\n")
 
-            return self.kvp_to_dict(sections)
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
 
         except Exception as e:
             self.logger.error(f"billing_req_to_dict: {e} while parsing {data}")
@@ -306,7 +376,10 @@ class AllnetServlet:
             unzipped = zlib.decompress(zipped)
             sections = unzipped.decode("utf-8").split("\r\n")
 
-            return self.kvp_to_dict(sections)
+            ret = []
+            for x in sections:
+                ret.append(dict(urllib.parse.parse_qsl(x)))
+            return ret
 
         except Exception as e:
             self.logger.error(f"allnet_req_to_dict: {e} while parsing {data}")
@@ -315,7 +388,7 @@ class AllnetServlet:
     def dict_to_http_form_string(
         self,
         data: List[Dict[str, Any]],
-        crlf: bool = False,
+        crlf: bool = True,
         trailing_newline: bool = True,
     ) -> Optional[str]:
         """
@@ -325,21 +398,19 @@ class AllnetServlet:
             urlencode = ""
             for item in data:
                 for k, v in item.items():
+                    if k is None or v is None:
+                        continue
                     urlencode += f"{k}={v}&"
-
                 if crlf:
                     urlencode = urlencode[:-1] + "\r\n"
                 else:
                     urlencode = urlencode[:-1] + "\n"
-
             if not trailing_newline:
                 if crlf:
                     urlencode = urlencode[:-2]
                 else:
                     urlencode = urlencode[:-1]
-
             return urlencode
-
         except Exception as e:
             self.logger.error(f"dict_to_http_form_string: {e} while parsing {data}")
             return None
@@ -349,64 +420,68 @@ class AllnetPowerOnRequest:
     def __init__(self, req: Dict) -> None:
         if req is None:
             raise AllnetRequestException("Request processing failed")
-        self.game_id: str = req.get("game_id", "")
+        self.game_id: str = req.get("game_id", None)
-        self.ver: str = req.get("ver", "")
+        self.ver: str = req.get("ver", None)
-        self.serial: str = req.get("serial", "")
+        self.serial: str = req.get("serial", None)
-        self.ip: str = req.get("ip", "")
+        self.ip: str = req.get("ip", None)
-        self.firm_ver: str = req.get("firm_ver", "")
+        self.firm_ver: str = req.get("firm_ver", None)
-        self.boot_ver: str = req.get("boot_ver", "")
+        self.boot_ver: str = req.get("boot_ver", None)
-        self.encode: str = req.get("encode", "")
+        self.encode: str = req.get("encode", "EUC-JP")
-        self.hops = int(req.get("hops", "0"))
+        self.hops = int(req.get("hops", "-1"))
-        self.format_ver = req.get("format_ver", "2")
+        self.format_ver = float(req.get("format_ver", "1.00"))
-        self.token = int(req.get("token", "0"))
+        self.token: str = req.get("token", "0")
 
-
+class AllnetPowerOnResponse:
-class AllnetPowerOnResponse3:
-    def __init__(self, token) -> None:
-        self.stat = 1
-        self.uri = ""
-        self.host = ""
-        self.place_id = "123"
-        self.name = ""
-        self.nickname = ""
-        self.region0 = "1"
-        self.region_name0 = "W"
-        self.region_name1 = ""
-        self.region_name2 = ""
-        self.region_name3 = ""
-        self.country = "JPN"
-        self.allnet_id = "123"
-        self.client_timezone = "+0900"
-        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
-            "%Y-%m-%dT%H:%M:%SZ"
-        )
-        self.setting = "1"
-        self.res_ver = "3"
-        self.token = str(token)
-
-
-class AllnetPowerOnResponse2:
     def __init__(self) -> None:
         self.stat = 1
         self.uri = ""
         self.host = ""
         self.place_id = "123"
-        self.name = "Test"
+        self.name = "ARTEMiS"
-        self.nickname = "Test123"
+        self.nickname = "ARTEMiS"
         self.region0 = "1"
         self.region_name0 = "W"
-        self.region_name1 = "X"
+        self.region_name1 = ""
-        self.region_name2 = "Y"
+        self.region_name2 = ""
-        self.region_name3 = "Z"
+        self.region_name3 = ""        
-        self.country = "JPN"
+        self.setting = "1"
         self.year = datetime.now().year
         self.month = datetime.now().month
         self.day = datetime.now().day
         self.hour = datetime.now().hour
         self.minute = datetime.now().minute
         self.second = datetime.now().second
-        self.setting = "1"
+
-        self.timezone = "+0900"
+class AllnetPowerOnResponse3(AllnetPowerOnResponse):
+    def __init__(self, token) -> None:
+        super().__init__()
+
+        # Added in v3
+        self.country = "JPN"
+        self.allnet_id = "123"
+        self.client_timezone = "+0900"
+        self.utc_time = datetime.now(tz=pytz.timezone("UTC")).strftime(
+            "%Y-%m-%dT%H:%M:%SZ"
+        )
+        self.res_ver = "3"
+        self.token = token
+
+        # Removed in v3
+        self.year = None
+        self.month = None
+        self.day = None
+        self.hour = None
+        self.minute = None
+        self.second = None
+
+
+class AllnetPowerOnResponse2(AllnetPowerOnResponse):
+    def __init__(self) -> None:
+        super().__init__()
+
+        # Added in v2
+        self.country = "JPN"
+        self.timezone = "+09:00"
         self.res_class = "PowerOnResponseV2"
 
 
@@ -419,7 +494,7 @@ class AllnetDownloadOrderRequest:
 
 
 class AllnetDownloadOrderResponse:
-    def __init__(self, stat: int = 1, serial: str = "", uri: str = "null") -> None:
+    def __init__(self, stat: int = 1, serial: str = "", uri: str = "") -> None:
         self.stat = stat
         self.serial = serial
         self.uri = uri

core/config.py

@@ -36,6 +36,12 @@ class ServerConfig:
             self.__config, "core", "server", "is_develop", default=True
         )
 
+    @property
+    def threading(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "server", "threading", default=False
+        )
+
     @property
     def log_dir(self) -> str:
         return CoreConfig.get_config_field(
@@ -188,6 +194,12 @@ class AllnetConfig:
             self.__config, "core", "allnet", "allow_online_updates", default=False
         )
 
+    @property
+    def update_cfg_folder(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "update_cfg_folder", default=""
+        )
+
 
 class BillingConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:

core/data/database.py

@@ -1,5 +1,5 @@
 import logging, coloredlogs
-from typing import Optional
+from typing import Optional, Dict, List
 from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
@@ -15,6 +15,13 @@ from core.utils import Utils
 
 
 class Data:
+    current_schema_version = 4
+    engine = None
+    session = None
+    user = None
+    arcade = None
+    card = None
+    base = None
     def __init__(self, cfg: CoreConfig) -> None:
         self.config = cfg
 
@@ -24,22 +31,32 @@ class Data:
         else:
             self.__url = f"{self.config.database.protocol}://{self.config.database.username}:{self.config.database.password}@{self.config.database.host}/{self.config.database.name}?charset=utf8mb4"
 
-        self.__engine = create_engine(self.__url, pool_recycle=3600)
+        if Data.engine is None:
-        session = sessionmaker(bind=self.__engine, autoflush=True, autocommit=True)
+            Data.engine = create_engine(self.__url, pool_recycle=3600)
-        self.session = scoped_session(session)
+            self.__engine = Data.engine
 
-        self.user = UserData(self.config, self.session)
+        if Data.session is None:
-        self.arcade = ArcadeData(self.config, self.session)
+            s = sessionmaker(bind=Data.engine, autoflush=True, autocommit=True)
-        self.card = CardData(self.config, self.session)
+            Data.session = scoped_session(s)
-        self.base = BaseData(self.config, self.session)
+
-        self.schema_ver_latest = 4
+        if Data.user is None:
+            Data.user = UserData(self.config, self.session)
+        
+        if Data.arcade is None:
+            Data.arcade = ArcadeData(self.config, self.session)
+        
+        if Data.card is None:
+            Data.card = CardData(self.config, self.session)
+        
+        if Data.base is None:
+            Data.base = BaseData(self.config, self.session)
 
-        log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
-        log_fmt = logging.Formatter(log_fmt_str)
         self.logger = logging.getLogger("database")
 
         # Prevent the logger from adding handlers multiple times
         if not getattr(self.logger, "handler_set", None):            
+            log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
             fileHandler = TimedRotatingFileHandler(
                 "{0}/{1}.log".format(self.config.server.log_dir, "db"),
                 encoding="utf-8",
@@ -71,7 +88,9 @@ class Data:
         games = Utils.get_all_titles()
         for game_dir, game_mod in games.items():
             try:
-                if hasattr(game_mod, "database") and hasattr(game_mod, "current_schema_version"):
+                if hasattr(game_mod, "database") and hasattr(
+                    game_mod, "current_schema_version"
+                ):
                     game_mod.database(self.config)
                 metadata.create_all(self.__engine.connect())
 
@@ -84,8 +103,8 @@ class Data:
                     f"Could not load database schema from {game_dir} - {e}"
                 )
 
-        self.logger.info(f"Setting base_schema_ver to {self.schema_ver_latest}")
+        self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
-        self.base.set_schema_ver(self.schema_ver_latest)
+        self.base.set_schema_ver(self.current_schema_version)
 
         self.logger.info(
             f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
@@ -129,9 +148,32 @@ class Data:
 
         self.create_database()
 
-    def migrate_database(self, game: str, version: int, action: str) -> None:
+    def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
         old_ver = self.base.get_schema_ver(game)
         sql = ""
+        if version is None:
+            if not game == "CORE":
+                titles = Utils.get_all_titles()
+
+                for folder, mod in titles.items():
+                    if not mod.game_codes[0] == game:
+                        continue
+
+                    if hasattr(mod, "current_schema_version"):
+                        version = mod.current_schema_version
+
+                    else:
+                        self.logger.warn(
+                            f"current_schema_version not found for {folder}"
+                        )
+
+            else:
+                version = self.current_schema_version
+
+        if version is None:
+            self.logger.warn(
+                f"Could not determine latest version for {game}, please specify --version"
+            )
 
         if old_ver is None:
             self.logger.error(
@@ -263,17 +305,53 @@ class Data:
                 self.user.delete_user(user["id"])
 
     def autoupgrade(self) -> None:
-        all_games = self.base.get_all_schema_vers()
+        all_game_versions = self.base.get_all_schema_vers()
-        if all_games is None:
+        if all_game_versions is None:
             self.logger.warn("Failed to get schema versions")
+            return
 
-        for x in all_games:
+        all_games = Utils.get_all_titles()
+        all_games_list: Dict[str, int] = {}
+        for _, mod in all_games.items():
+            if hasattr(mod, "current_schema_version"):
+                all_games_list[mod.game_codes[0]] = mod.current_schema_version
+
+        for x in all_game_versions:
+            failed = False
             game = x["game"].upper()
-            update_ver = 1
+            update_ver = int(x["version"])
-            for y in range(2, 100):
+            latest_ver = all_games_list.get(game, 1)
-                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
+            if game == "CORE":
-                    update_ver = y
+                latest_ver = self.current_schema_version
-                else:
-                    break
 
-            self.migrate_database(game, update_ver, "upgrade")
+            if update_ver == latest_ver:
+                self.logger.info(f"{game} is already latest version")
+                continue
+
+            for y in range(update_ver + 1, latest_ver + 1):
+                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
+                    with open(
+                        f"core/data/schema/versions/{game}_{y}_upgrade.sql",
+                        "r",
+                        encoding="utf-8",
+                    ) as f:
+                        sql = f.read()
+
+                    result = self.base.execute(sql)
+                    if result is None:
+                        self.logger.error(
+                            f"Error execuing sql script for game {game} v{y}!"
+                        )
+                        failed = True
+                        break
+                else:
+                    self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
+                    failed = True
+
+            if not failed:
+                self.base.set_schema_ver(latest_ver, game)
+    
+    def show_versions(self) -> None:
+        all_game_versions = self.base.get_all_schema_vers()
+        for ver in all_game_versions:
+            self.logger.info(f"{ver['game']} -> v{ver['version']}")

core/data/schema/base.py

@@ -47,7 +47,7 @@ class BaseData:
         res = None
 
         try:
-            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())} || {opts}")
+            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}")
             res = self.conn.execute(text(sql), opts)
 
         except SQLAlchemyError as e:

core/data/schema/user.py

@@ -80,6 +80,9 @@ class UserData(BaseData):
         if usr["password"] is None:
             return False
         
+        if passwd is None or not passwd:
+            return False
+
         return bcrypt.checkpw(passwd, usr["password"].encode())
 
     def reset_autoincrement(self, ai_value: int) -> None:

core/data/schema/versions/SBZV_4_rollback.sql

@@ -0,0 +1,9 @@
+ALTER TABLE diva_profile
+    DROP cnp_cid,
+    DROP cnp_val,
+    DROP cnp_rr,
+    DROP cnp_sp,
+    DROP btn_se_eqp,
+    DROP sld_se_eqp,
+    DROP chn_sld_se_eqp,
+    DROP sldr_tch_se_eqp;

core/data/schema/versions/SBZV_5_upgrade.sql

@@ -0,0 +1,9 @@
+ALTER TABLE diva_profile
+    ADD cnp_cid INT NOT NULL DEFAULT -1,
+    ADD cnp_val INT NOT NULL DEFAULT -1,
+    ADD cnp_rr INT NOT NULL DEFAULT -1,
+    ADD cnp_sp VARCHAR(255) NOT NULL DEFAULT "",
+    ADD btn_se_eqp INT NOT NULL DEFAULT -1,
+    ADD sld_se_eqp INT NOT NULL DEFAULT -1,
+    ADD chn_sld_se_eqp INT NOT NULL DEFAULT -1,
+    ADD sldr_tch_se_eqp INT NOT NULL DEFAULT -1;

core/data/schema/versions/SDBT_3_rollback.sql

@@ -0,0 +1,30 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    DROP COLUMN regionId,
+    DROP COLUMN machineType;
+
+ALTER TABLE chuni_static_events
+    DROP COLUMN startDate;
+
+ALTER TABLE chuni_profile_data
+    DROP COLUMN rankUpChallengeResults;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN presetId id INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD PRIMARY KEY(id);
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_uk UNIQUE(id, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY(presetId)
+    REFERENCES chuni_static_login_bonus_preset(id) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_4_upgrade.sql

@@ -0,0 +1,29 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    ADD COLUMN regionId INT,
+    ADD COLUMN machineType INT;
+
+ALTER TABLE chuni_static_events
+    ADD COLUMN startDate TIMESTAMP NOT NULL DEFAULT current_timestamp();
+
+ALTER TABLE chuni_profile_data
+    ADD COLUMN rankUpChallengeResults JSON;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN id presetId INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP INDEX chuni_static_login_bonus_preset_uk;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_pk PRIMARY KEY (presetId, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY (presetId, version)
+    REFERENCES chuni_static_login_bonus_preset(presetId, version) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDDT_4_rollback.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ongeki_static_events
+DROP COLUMN startDate;

core/data/schema/versions/SDDT_5_upgrade.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ongeki_static_events
+ADD COLUMN startDate TIMESTAMP NOT NULL DEFAULT current_timestamp();

core/data/schema/versions/SDEZ_4_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP DEFAULT "2018-01-01 00:00:00.0",
+    CHANGE COLUMN endDate endDate TIMESTAMP DEFAULT "2038-01-01 00:00:00.0";

core/data/schema/versions/SDEZ_5_rollback.sql

@@ -0,0 +1,78 @@
+DELETE FROM mai2_static_event WHERE version < 13;
+UPDATE mai2_static_event SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_music WHERE version < 13;
+UPDATE mai2_static_music SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_ticket WHERE version < 13;
+UPDATE mai2_static_ticket SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_static_cards WHERE version < 13;
+UPDATE mai2_static_cards SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_detail WHERE version < 13;
+UPDATE mai2_profile_detail SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_extend WHERE version < 13;
+UPDATE mai2_profile_extend SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_option WHERE version < 13;
+UPDATE mai2_profile_option SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_ghost WHERE version < 13;
+UPDATE mai2_profile_ghost SET version = version - 13 WHERE version >= 13;
+
+DELETE FROM mai2_profile_rating WHERE version < 13;
+UPDATE mai2_profile_rating SET version = version - 13 WHERE version >= 13;
+
+DROP TABLE maimai_score_best;
+DROP TABLE maimai_playlog;
+DROP TABLE maimai_profile_detail;
+DROP TABLE maimai_profile_option;
+DROP TABLE maimai_profile_web_option;
+DROP TABLE maimai_profile_grade_status;
+
+ALTER TABLE mai2_item_character DROP COLUMN point;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NOT NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NOT NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NOT NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NOT NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NOT NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NOT NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NOT NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rank int(11) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NOT NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NOT NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NOT NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NOT NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL;

core/data/schema/versions/SDEZ_5_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CHANGE COLUMN endDate endDate TIMESTAMP NOT NULL;

core/data/schema/versions/SDEZ_6_rollback.sql

@@ -0,0 +1 @@
+DROP TABLE aime.mai2_profile_consec_logins;

core/data/schema/versions/SDEZ_6_upgrade.sql

@@ -0,0 +1,62 @@
+UPDATE mai2_static_event SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_music SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_ticket SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_static_cards SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_detail  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_extend  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_option  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_ghost  SET version = version + 13 WHERE version < 1000;
+
+UPDATE mai2_profile_rating  SET version = version + 13 WHERE version < 1000;
+
+ALTER TABLE mai2_item_character ADD point int(11) NULL;
+
+ALTER TABLE mai2_item_card MODIFY COLUMN cardId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN cardTypeId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN charaId int(11) NULL;
+ALTER TABLE mai2_item_card MODIFY COLUMN mapId int(11) NULL;
+
+ALTER TABLE mai2_item_character MODIFY COLUMN characterId int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN level int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN awakening int(11) NULL;
+ALTER TABLE mai2_item_character MODIFY COLUMN useCount int(11) NULL;
+
+ALTER TABLE mai2_item_charge MODIFY COLUMN chargeId int(11) NULL;
+ALTER TABLE mai2_item_charge MODIFY COLUMN stock int(11) NULL;
+
+ALTER TABLE mai2_item_favorite MODIFY COLUMN itemKind int(11) NULL;
+
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN seasonId int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rank int(11) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN rewardGet tinyint(1) NULL;
+ALTER TABLE mai2_item_friend_season_ranking MODIFY COLUMN userName varchar(8) NULL;
+
+ALTER TABLE mai2_item_item MODIFY COLUMN itemId int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN itemKind int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN stock int(11) NULL;
+ALTER TABLE mai2_item_item MODIFY COLUMN isValid tinyint(1) NULL;
+
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN bonusId int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN point int(11) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isCurrent tinyint(1) NULL;
+ALTER TABLE mai2_item_login_bonus MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_map MODIFY COLUMN mapId int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN distance int(11) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isLock tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isClear tinyint(1) NULL;
+ALTER TABLE mai2_item_map MODIFY COLUMN isComplete tinyint(1) NULL;
+
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printDate timestamp DEFAULT current_timestamp() NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN serialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN placeId int(11) NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN clientId varchar(11) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;
+ALTER TABLE mai2_item_print_detail MODIFY COLUMN printerSerialId varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL;

core/data/schema/versions/SDEZ_7_upgrade.sql

@@ -0,0 +1,9 @@
+CREATE TABLE `mai2_profile_consec_logins` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `user` int(11) NOT NULL,
+  `version` int(11) NOT NULL,
+  `logins` int(11) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `mai2_profile_consec_logins_uk` (`user`,`version`),
+  CONSTRAINT `mai2_profile_consec_logins_ibfk_1` FOREIGN KEY (`user`) REFERENCES `aime_user` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

core/data/schema/versions/SDFE_3_rollback.sql

@@ -0,0 +1,2 @@
+SET FOREIGN_KEY_CHECKS=0;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDFE_4_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE wacca_profile DROP COLUMN playcount_time_free;

core/data/schema/versions/SDFE_4_upgrade.sql

@@ -0,0 +1 @@
+DELETE FROM wacca_item WHERE type=17 AND item_id=312002;

core/data/schema/versions/SDFE_5_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE wacca_profile ADD playcount_time_free int(11) DEFAULT 0 NULL AFTER playcount_stageup;

core/frontend.py

@@ -71,8 +71,11 @@ class FrontendServlet(resource.Resource):
                     game_fe = game_mod.frontend(cfg, self.environment, config_dir)
                     self.game_list.append({"url": game_dir, "name": game_fe.nav_name})
                     fe_game.putChild(game_dir.encode(), game_fe)
-                except:
+
-                    raise
+                except Exception as e:
+                    self.logger.error(
+                        f"Failed to import frontend from {game_dir} because {e}"
+                    )
 
         self.environment.globals["game_list"] = self.game_list
         self.putChild(b"gate", FE_Gate(cfg, self.environment))
@@ -179,7 +182,7 @@ class FE_Gate(FE_Base):
             access_code: str = request.args[b"access_code"][0].decode()
             username: str = request.args[b"username"][0]
             email: str = request.args[b"email"][0].decode()
-            passwd: str = request.args[b"passwd"][0]
+            passwd: bytes = request.args[b"passwd"][0]
 
             uid = self.data.card.get_user_id_from_card(access_code)
             if uid is None:
@@ -194,7 +197,7 @@ class FE_Gate(FE_Base):
             if result is None:
                 return redirectTo(b"/gate?e=3", request)
 
-            if not self.data.user.check_password(uid, passwd.encode()):
+            if not self.data.user.check_password(uid, passwd):
                 return redirectTo(b"/gate", request)
 
             return redirectTo(b"/user", request)
@@ -225,8 +228,21 @@ class FE_User(FE_Base):
         if usr_sesh.userId == 0:
             return redirectTo(b"/gate", request)
         
+        cards = self.data.card.get_user_cards(usr_sesh.userId)
+        user = self.data.user.get_user(usr_sesh.userId)
+        card_data = []
+        for c in cards:
+            if c['is_locked']:
+                status = 'Locked'
+            elif c['is_banned']:
+                status = 'Banned'
+            else:
+                status = 'Active'
+            
+            card_data.append({'access_code': c['access_code'], 'status': status})
+
         return template.render(
-            title=f"{self.core_config.server.name} | Account", sesh=vars(usr_sesh)
+            title=f"{self.core_config.server.name} | Account", sesh=vars(usr_sesh), cards=card_data, username=user['username']
         ).encode("utf-16")
 
 

core/frontend/user/index.jinja

@@ -1,4 +1,31 @@
 {% extends "core/frontend/index.jinja" %}
 {% block content %}
-<h1>testing</h1>
+<h1>Management for {{ username }}</h1>
+<h2>Cards <button class="btn btn-success" data-bs-toggle="modal" data-bs-target="#card_add">Add</button></h2>
+<ul>
+{% for c in cards %}
+<li>{{ c.access_code }}: {{ c.status }} <button class="btn-danger btn">Delete</button></li>
+{% endfor %}
+</ul>
+<div class="modal fade" id="card_add" tabindex="-1" aria-labelledby="card_add_label" aria-hidden="true">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h1 class="modal-title fs-5" id="card_add_label">Add Card</h1>
+        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+      </div>
+      <div class="modal-body">
+        HOW TO:<br>
+        Scan your card on any networked game and press the "View Access Code" button (varies by game) and enter the 20 digit code below.<br>
+        !!FOR AMUSEIC CARDS: DO NOT ENTER THE CODE SHOWN ON THE BACK OF THE CARD ITSELF OR IT WILL NOT WORK!!
+        <p /><label for="card_add_frm_access_code">Access Code:&nbsp;</label><input id="card_add_frm_access_code" maxlength="20" type="text" required> 
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-primary">Add</button>
+        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+      </div>
+    </div>
+  </div>
+</div>
+
 {% endblock content %}

core/frontend/widgets/topbar.jinja

@@ -4,7 +4,7 @@
 <div style="background: #333; color: #f9f9f9; width: 80%; height: 50px; line-height: 50px; padding-left: 10px; float: left;">
     <a href=/><button class="btn btn-primary">Home</button></a>&nbsp;
     {% for game in game_list %}    
-    <a href=game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
+    <a href=/game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
     {% endfor %}
 </div>
 </div>

core/mucha.py

@@ -33,8 +33,8 @@ class MuchaServlet:
         self.logger.addHandler(fileHandler)
         self.logger.addHandler(consoleHandler)
 
-        self.logger.setLevel(logging.INFO)
+        self.logger.setLevel(cfg.mucha.loglevel)
-        coloredlogs.install(level=logging.INFO, logger=self.logger, fmt=log_fmt_str)
+        coloredlogs.install(level=cfg.mucha.loglevel, logger=self.logger, fmt=log_fmt_str)
 
         all_titles = Utils.get_all_titles()
 
@@ -46,9 +46,7 @@ class MuchaServlet:
                 if enabled:
                     self.mucha_registry.append(game_cd)
 
-        self.logger.info(
+        self.logger.info(f"Serving {len(self.mucha_registry)} games")
-            f"Serving {len(self.mucha_registry)} games"
-        )
 
     def handle_boardauth(self, request: Request, _: Dict) -> bytes:
         req_dict = self.mucha_preprocess(request.content.getvalue())
@@ -62,9 +60,7 @@ class MuchaServlet:
 
         req = MuchaAuthRequest(req_dict)
         self.logger.debug(f"Mucha request {vars(req)}")
-        self.logger.info(
+        self.logger.info(f"Boardauth request from {client_ip} for {req.gameVer}")
-            f"Boardauth request from {client_ip} for {req.gameVer}"
-        )
 
         if req.gameCd not in self.mucha_registry:
             self.logger.warn(f"Unknown gameCd {req.gameCd}")
@@ -92,9 +88,7 @@ class MuchaServlet:
 
         req = MuchaUpdateRequest(req_dict)
         self.logger.debug(f"Mucha request {vars(req)}")
-        self.logger.info(
+        self.logger.info(f"Updatecheck request from {client_ip} for {req.gameVer}")
-            f"Updatecheck request from {client_ip} for {req.gameVer}"
-        )
 
         if req.gameCd not in self.mucha_registry:
             self.logger.warn(f"Unknown gameCd {req.gameCd}")

core/title.py

@@ -84,7 +84,7 @@ class TitleServlet:
             request.setResponseCode(405)
             return b""
 
-        return index.render_GET(request, endpoints["version"], endpoints["endpoint"])
+        return index.render_GET(request, int(endpoints["version"]), endpoints["endpoint"])
 
     def render_POST(self, request: Request, endpoints: dict) -> bytes:
         code = endpoints["game"]

core/utils.py

@@ -16,6 +16,9 @@ class Utils:
                 if not dir.startswith("__"):
                     try:
                         mod = importlib.import_module(f"titles.{dir}")
+                        if hasattr(mod, "game_codes") and hasattr(
+                            mod, "index"
+                        ):  # Minimum required to function
                             ret[dir] = mod
 
                     except ImportError as e:
@@ -25,4 +28,8 @@ class Utils:
 
     @classmethod
     def get_ip_addr(cls, req: Request) -> str:
-        return req.getAllHeaders()[b"x-forwarded-for"].decode() if b"x-forwarded-for" in req.getAllHeaders() else req.getClientAddress().host
+        return (
+            req.getAllHeaders()[b"x-forwarded-for"].decode()
+            if b"x-forwarded-for" in req.getAllHeaders()
+            else req.getClientAddress().host
+        )

dbutils.py

@@ -1,5 +1,6 @@
 import yaml
 import argparse
+import logging
 from core.config import CoreConfig
 from core.data import Data
 from os import path, mkdir, access, W_OK
@@ -32,7 +33,9 @@ if __name__ == "__main__":
 
     cfg = CoreConfig()
     if path.exists(f"{args.config}/core.yaml"):
-        cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
+        cfg_dict = yaml.safe_load(open(f"{args.config}/core.yaml"))
+        cfg_dict.get("database", {})["loglevel"] = "info"
+        cfg.update(cfg_dict)
 
     if not path.exists(cfg.server.log_dir):
         mkdir(cfg.server.log_dir)
@@ -45,7 +48,6 @@ if __name__ == "__main__":
 
     data = Data(cfg)
 
-
     if args.action == "create":
         data.create_database()
 
@@ -54,15 +56,22 @@ if __name__ == "__main__":
 
     elif args.action == "upgrade" or args.action == "rollback":
         if args.version is None:
-            data.logger.error("Must set game and version to migrate to")
+            data.logger.warn("No version set, upgrading to latest")
-            exit(0)
 
         if args.game is None:
-            data.logger.info("No game set, upgrading core schema")
+            data.logger.warn("No game set, upgrading core schema")
-            data.migrate_database("CORE", int(args.version), args.action)
+            data.migrate_database(
+                "CORE",
+                int(args.version) if args.version is not None else None,
+                args.action,
+            )
 
         else:
-            data.migrate_database(args.game, int(args.version), args.action)
+            data.migrate_database(
+                args.game,
+                int(args.version) if args.version is not None else None,
+                args.action,
+            )
 
     elif args.action == "autoupgrade":
         data.autoupgrade()
@@ -76,4 +85,7 @@ if __name__ == "__main__":
     elif args.action == "cleanup":
         data.delete_hanging_users()
     
+    elif args.action == "version":
+        data.show_versions()
+
     data.logger.info("Done")

docker-compose.yml

@@ -0,0 +1,57 @@
+version: "3.9"
+services:
+  app:
+    hostname: ma.app
+    build: .
+    volumes:
+      - ./aime:/app/aime
+
+    environment:
+      CFG_DEV: 1
+      CFG_CORE_SERVER_HOSTNAME: 0.0.0.0
+      CFG_CORE_DATABASE_HOST: ma.db
+      CFG_CORE_MEMCACHED_HOSTNAME: ma.memcached
+      CFG_CORE_AIMEDB_KEY: keyhere
+      CFG_CHUNI_SERVER_LOGLEVEL: debug
+
+    ports:
+    - "80:80"
+    - "8443:8443"
+    - "22345:22345"
+
+    - "8080:8080"
+    - "8090:8090"
+
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    hostname: ma.db
+    image: mysql:8.0.31-debian
+    environment:
+      MYSQL_DATABASE: aime
+      MYSQL_USER: aime
+      MYSQL_PASSWORD: aime
+      MYSQL_ROOT_PASSWORD: AimeRootPassword
+    healthcheck:
+      test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"]
+      timeout: 5s
+      retries: 5
+
+
+  memcached:
+    hostname: ma.memcached
+    image: memcached:1.6.17-bullseye
+
+  phpmyadmin:
+    hostname: ma.phpmyadmin
+    image: phpmyadmin:latest
+    environment:
+      PMA_HOSTS: ma.db
+      PMA_USER: root
+      PMA_PASSWORD: AimeRootPassword
+      APACHE_PORT: 8080
+    ports:
+    - "8080:8080"
+

docs/config.md

@@ -5,6 +5,7 @@
 - `allow_unregistered_serials`: Allows games that do not have registered keychips to connect and authenticate. Disable to restrict who can connect to your server. Recomended to disable for production setups. Default `True`
 - `name`: Name for the server, used by some games in their default MOTDs. Default `ARTEMiS`
 - `is_develop`: Flags that the server is a development instance without a proxy standing in front of it. Setting to `False` tells the server not to listen for SSL, because the proxy should be handling all SSL-related things, among other things. Default `True`
+- `threading`: Flags that `reactor.run` should be called via the `Thread` standard library. May provide a speed boost, but removes the ability to kill the server via `Ctrl + C`. Default: `False`
 - `log_dir`: Directory to store logs. Server MUST have read and write permissions to this directory or you will have issues. Default `logs`
 ## Title
 - `loglevel`: Logging level for the title server. Default `info`

docs/game_specific_info.md

@@ -9,42 +9,44 @@ using the megaime database. Clean installations always create the latest databas
 # Table of content
 
 - [Supported Games](#supported-games)
-    - [Chunithm](#chunithm)
+    - [CHUNITHM](#chunithm)
     - [crossbeats REV.](#crossbeats-rev)
     - [maimai DX](#maimai-dx)
     - [O.N.G.E.K.I.](#o-n-g-e-k-i)
     - [Card Maker](#card-maker)
     - [WACCA](#wacca)
+    - [Sword Art Online Arcade](#sao)
 
 
 # Supported Games
 
 Games listed below have been tested and confirmed working.
 
-## Chunithm
+## CHUNITHM
 
 ### SDBT
 
 | Version ID | Version Name          |
-|------------|--------------------|
+|------------|-----------------------|
-| 0          | Chunithm           |
+| 0          | CHUNITHM              |
-| 1          | Chunithm+          |
+| 1          | CHUNITHM PLUS         |
-| 2          | Chunithm Air       |
+| 2          | CHUNITHM AIR          |
-| 3          | Chunithm Air +     |
+| 3          | CHUNITHM AIR PLUS     |
-| 4          | Chunithm Star      |
+| 4          | CHUNITHM STAR         |
-| 5          | Chunithm Star +    |
+| 5          | CHUNITHM STAR PLUS    |
-| 6          | Chunithm Amazon    |
+| 6          | CHUNITHM AMAZON       |
-| 7          | Chunithm Amazon +  |
+| 7          | CHUNITHM AMAZON PLUS  |
-| 8          | Chunithm Crystal   |
+| 8          | CHUNITHM CRYSTAL      |
-| 9          | Chunithm Crystal + |
+| 9          | CHUNITHM CRYSTAL PLUS |
-| 10         | Chunithm Paradise  |
+| 10         | CHUNITHM PARADISE     |
 
 ### SDHD/SDBT
 
 | Version ID | Version Name        |
-|------------|-----------------|
+|------------|---------------------|
-| 11         | Chunithm New!!  |
+| 11         | CHUNITHM NEW!!      |
-| 12         | Chunithm New!!+ |
+| 12         | CHUNITHM NEW PLUS!! |
+| 13         | CHUNITHM SUN        |
 
 
 ### Importer
@@ -60,14 +62,33 @@ The importer for Chunithm will import: Events, Music, Charge Items and Avatar Ac
 ### Database upgrade
 
 Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see
-which version is the latest, f.e. `SDBT_3_upgrade.sql`. In order to upgrade to version 3 in this case you need to
+which version is the latest, f.e. `SDBT_4_upgrade.sql`. In order to upgrade to version 4 in this case you need to
 perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDBT --version 2 upgrade
+python dbutils.py --game SDBT upgrade
-python dbutils.py --game SDBT --version 3 upgrade
 ```
 
+### Online Battle
+
+**Only matchmaking (with your imaginary friends) is supported! Online Battle does not (yet?) work!**
+
+The first person to start the Online Battle (now called host) will create a "matching room" with a given `roomId`, after that max 3 other people can join the created room.
+Non used slots during the matchmaking will be filled with CPUs after the timer runs out.
+As soon as a new member will join the room the timer will jump back to 60 secs again.
+Sending those 4 messages to all other users is also working properly.
+In order to use the Online Battle every user needs the same ICF, same rom version and same data version!
+If a room is full a new room will be created if another user starts an Online Battle.
+After a failed Online Battle the room will be deleted. The host is used for the timer countdown, so if the connection failes to the host the timer will stop and could create a "frozen" state.
+
+#### Information/Problems:
+
+- Online Battle uses UDP hole punching and opens port 50201?
+- `reflectorUri` seems related to that?
+- Timer countdown should be handled globally and not by one user
+- Game can freeze or can crash if someone (especially the host) leaves the matchmaking
+
+
 ## crossbeats REV.
 
 ### SDCA
@@ -106,36 +127,59 @@ Config file is located in `config/cxb.yaml`.
 
 ### SDEZ
 
-| Version ID | Version Name            |
+| Game Code | Version ID | Version Name            |
-|------------|-------------------------|
+|-----------|------------|-------------------------|
-| 0          | maimai DX               |
+
-| 1          | maimai DX PLUS          |
+
-| 2          | maimai DX Splash        |
+For versions pre-dx
-| 3          | maimai DX Splash PLUS   |
+| Game Code | Version ID | Version Name            |
-| 4          | maimai DX Universe      |
+|-----------|------------|-------------------------|
-| 5          | maimai DX Universe PLUS |
+| SBXL      | 0          | maimai                  |
-| 6          | maimai DX Festival      |
+| SBXL      | 1          | maimai PLUS             |
+| SBZF      | 2          | maimai GreeN            |
+| SBZF      | 3          | maimai GreeN PLUS       |
+| SDBM      | 4          | maimai ORANGE           |
+| SDBM      | 5          | maimai ORANGE PLUS      |
+| SDCQ      | 6          | maimai PiNK             |
+| SDCQ      | 7          | maimai PiNK PLUS        |
+| SDDK      | 8          | maimai MURASAKI         |
+| SDDK      | 9          | maimai MURASAKI PLUS    |
+| SDDZ      | 10         | maimai MILK             |
+| SDDZ      | 11         | maimai MILK PLUS        |
+| SDEY      | 12         | maimai FiNALE           |
+| SDEZ      | 13         | maimai DX               |
+| SDEZ      | 14         | maimai DX PLUS          |
+| SDEZ      | 15         | maimai DX Splash        |
+| SDEZ      | 16         | maimai DX Splash PLUS   |
+| SDEZ      | 17         | maimai DX Universe      |
+| SDEZ      | 18         | maimai DX Universe PLUS |
+| SDEZ      | 19         | maimai DX Festival      |
 
 ### Importer
 
 In order to use the importer locate your game installation folder and execute:
-
+DX:
 ```shell
-python read.py --series SDEZ --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
+python read.py --series <Game Code> --version <Version ID> --binfolder /path/to/StreamingAssets --optfolder /path/to/game/option/folder
+```
+Pre-DX:
+```shell
+python read.py --series <Game Code> --version <Version ID> --binfolder /path/to/data --optfolder /path/to/patch/data
 ```
-
 The importer for maimai DX will import Events, Music and Tickets.
 
-**NOTE: It is required to use the importer because the game will
+The importer for maimai Pre-DX will import Events and Music. Not all games will have patch data. Milk - Finale have file encryption, and need an AES key. That key is not provided by the developers. For games that do use encryption, provide the key, as a hex string, with the `--extra` flag. Ex `--extra 00112233445566778899AABBCCDDEEFF`
-crash without Events!**
+
+**Important: It is required to use the importer because some games may not function properly or even crash without Events!**
 
 ### Database upgrade
 
 Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDEZ_2_upgrade.sql`. In order to upgrade to version 2 in this case you need to perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDEZ --version 2 upgrade
+python dbutils.py --game SDEZ upgrade
 ```
+Pre-Dx uses the same database as DX, so only upgrade using the SDEZ game code!
 
 ## Hatsune Miku Project Diva
 
@@ -175,9 +219,7 @@ which version is the latest, f.e. `SBZV_4_upgrade.sql`. In order to upgrade to v
 perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SBZV --version 2 upgrade
+python dbutils.py --game SBZV upgrade
-python dbutils.py --game SBZV --version 3 upgrade
-python dbutils.py --game SBZV --version 4 upgrade
 ```
 
 ## O.N.G.E.K.I.
@@ -188,12 +230,12 @@ python dbutils.py --game SBZV --version 4 upgrade
 |------------|----------------------------|
 | 0          | O.N.G.E.K.I.               |
 | 1          | O.N.G.E.K.I. +             |
-| 2          | O.N.G.E.K.I. Summer        |
+| 2          | O.N.G.E.K.I. SUMMER        |
-| 3          | O.N.G.E.K.I. Summer +      |
+| 3          | O.N.G.E.K.I. SUMMER +      |
-| 4          | O.N.G.E.K.I. Red           |
+| 4          | O.N.G.E.K.I. R.E.D.        |
-| 5          | O.N.G.E.K.I. Red +         |
+| 5          | O.N.G.E.K.I. R.E.D. +      |
-| 6          | O.N.G.E.K.I. Bright        |
+| 6          | O.N.G.E.K.I. bright        |
-| 7          | O.N.G.E.K.I. Bright Memory |
+| 7          | O.N.G.E.K.I. bright MEMORY |
 
 
 ### Importer
@@ -225,9 +267,7 @@ which version is the latest, f.e. `SDDT_4_upgrade.sql`. In order to upgrade to v
 perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDDT --version 2 upgrade
+python dbutils.py --game SDDT upgrade
-python dbutils.py --game SDDT --version 3 upgrade
-python dbutils.py --game SDDT --version 4 upgrade
 ```
 
 ## Card Maker
@@ -236,21 +276,21 @@ python dbutils.py --game SDDT --version 4 upgrade
 
 | Version ID | Version Name    |
 |------------|-----------------|
-| 0          | Card Maker 1.34 |
+| 0          | Card Maker 1.30 |
 | 1          | Card Maker 1.35 |
 
 
 ### Support status
 
-* Card Maker 1.34:
+* Card Maker 1.30:
-  * Chunithm New!!: Yes
+  * CHUNITHM NEW!!: Yes
-  * maimai DX Universe: Yes
+  * maimai DX UNiVERSE: Yes
-  * O.N.G.E.K.I. Bright: Yes
+  * O.N.G.E.K.I. bright: Yes
 
 * Card Maker 1.35:
-  * Chunithm New!!+: Yes
+  * CHUNITHM SUN: Yes (NEW PLUS!! up to A032)
-  * maimai DX Universe PLUS: Yes
+  * maimai DX FESTiVAL: Yes (up to A035) (UNiVERSE PLUS up to A031)
-  * O.N.G.E.K.I. Bright Memory: Yes
+  * O.N.G.E.K.I. bright MEMORY: Yes
 
 
 ### Importer
@@ -268,19 +308,54 @@ python read.py --series SDED --version <version ID> --binfolder titles/cm/cm_dat
 python read.py --series SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
 ```
 
-Also make sure to import all maimai and Chunithm data as well:
+Also make sure to import all maimai DX and CHUNITHM data as well:
 
 ```shell
 python read.py --series SDED --version <version ID> --binfolder /path/to/cardmaker/CardMaker_Data
 ```
 
-The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai/Chunithm) and the hardcoded
+The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai DX/CHUNITHM) and the hardcoded
 Cards for each Gacha (O.N.G.E.K.I. only).
 
 **NOTE: Without executing the importer Card Maker WILL NOT work!**
 
 
-### O.N.G.E.K.I. Gachas
+### Config setup
+
+Make sure to update your `config/cardmaker.yaml` with the correct version for each game. To get the current version required to run a specific game, open every opt (Axxx) folder descending until you find all three folders:
+
+- `MU3`: O.N.G.E.K.I.
+- `MAI`: maimai DX
+- `CHU`: CHUNITHM
+
+Inside each folder is a `DataConfig.xml` file, for example:
+
+`MU3/DataConfig.xml`:
+```xml
+  <cardMakerVersion>
+    <major>1</major>
+    <minor>35</minor>
+    <release>3</release>
+  </cardMakerVersion>
+```
+
+Now update your `config/cardmaker.yaml` with the correct version number, for example:
+
+```yaml
+version:
+  1: # Card Maker 1.35
+    ongeki: 1.35.03
+```
+
+For now you also need to update your `config/ongeki.yaml` with the correct version number, for example:
+
+```yaml
+version:
+  7: # O.N.G.E.K.I. bright MEMORY
+    card_maker: 1.35.03
+```
+
+### O.N.G.E.K.I.
 
 Gacha "無料ガチャ" can only pull from the free cards with the following probabilities: 94%: R, 5% SR and 1% chance of
 getting an SSR card
@@ -293,20 +368,24 @@ and 3% chance of getting an SSR card
 All other (limited) gachas can pull from every card added to ongeki_static_cards but with the promoted cards
 (click on the green button under the banner) having a 10 times higher chance to get pulled
 
-### Chunithm Gachas
+### CHUNITHM
 
-All cards in Chunithm (basically just the characters) have the same rarity to it just pulls randomly from all cards
+All cards in CHUNITHM (basically just the characters) have the same rarity to it just pulls randomly from all cards
 from a given gacha but made sure you cannot pull the same card twice in the same 5 times gacha roll.
 
+### maimai DX
+
+Printed maimai DX cards: Freedom (`cardTypeId=6`) or Gold Pass (`cardTypeId=4`) can now be selected during the login process. You can only have ONE Freedom and ONE Gold Pass active at a given time. The cards will expire after 15 days.
+
+Thanks GetzeAvenue for the `selectedCardList` rarity hint!
+
 ### Notes
 
-Card Maker 1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35 will only load an O.N.G.E.K.I.
+Card Maker 1.30-1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35+ will only load an O.N.G.E.K.I.
 Bright Memory profile (1.35).
-The gachas inside the `ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
+The gachas inside the `config/ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
 Gacha IDs up to 1140 will be loaded for CM 1.34 and all gachas will be loaded for CM 1.35.
 
-**NOTE: There is currently no way to load/use the (printed) maimai DX cards!**
-
 ## WACCA
 
 ### SDFE
@@ -347,6 +426,91 @@ Config file is located in `config/wacca.yaml`.
 Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDFE_3_upgrade.sql`. In order to upgrade to version 3 in this case you need to perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDFE --version 2 upgrade
+python dbutils.py --game SDFE upgrade
-python dbutils.py --game SDFE --version 3 upgrade
 ```
+
+### VIP Rewards
+Below is a list of VIP rewards. Currently, VIP is not implemented, and thus these are not obtainable. These 23 rewards were distributed once per month for VIP users on the real network.
+
+	Plates:
+		211004 リッチ
+		211018 特盛えりざべす
+		211025 イースター
+		211026 特盛りりぃ
+		311004 ファンシー
+		311005 インカンテーション
+		311014 夜明け
+		311015 ネイビー
+		311016 特盛るーん
+	
+	Ring Colors:
+		203002 Gold Rushイエロー
+		203009 トロピカル
+		303005 ネイチャー
+	
+	Icons:
+		202020 どらみんぐ
+		202063 ユニコーン
+		202086 ゴリラ
+		302014 ローズ
+		302015 ファラオ
+		302045 肉球
+		302046 WACCA
+		302047 WACCA Lily
+		302048 WACCA Reverse
+	
+	Note Sound Effect:
+		205002 テニス
+		205008 シャワー
+		305003 タンバリンMk-Ⅱ
+
+## SAO
+
+### SDEW
+
+| Version ID | Version Name  |
+|------------|---------------|
+| 0          | SAO           |
+
+
+### Importer
+
+In order to use the importer locate your game installation folder and execute:
+
+```shell
+python read.py --series SDEW --version <version ID> --binfolder /path/to/game/extractedassets
+```
+
+The importer for SAO will import all items, heroes, support skills and titles data.
+
+### Config
+
+Config file is located in `config/sao.yaml`.
+
+| Option             | Info                                                                        |
+|--------------------|-----------------------------------------------------------------------------|
+| `hostname`         | Changes the server listening address for Mucha                              |
+| `port`             | Changes the listing port                                                    |
+| `auto_register`    | Allows the game to handle the automatic registration of new cards           |
+
+
+### Database upgrade
+
+Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDEW_1_upgrade.sql`. In order to upgrade to version 3 in this case you need to perform all previous updates as well:
+
+```shell
+python dbutils.py --game SDEW upgrade
+```
+
+### Notes
+- Defrag Match will crash at loading
+- Co-Op Online is not supported
+- Shop is not functionnal
+- Player title is currently static and cannot be changed in-game
+- QR Card Scanning currently only load a static hero
+
+### Credits for SAO support:
+
+- Midorica - Limited Network Support
+- Dniel97 - Helping with network base
+- tungnotpunk - Source

entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [[ -z "${CFG_DEV}" ]]; then
+    echo Production mode
+    python3 index.py
+else
+    echo Development mode
+    python3 dbutils.py create
+    nodemon -w aime --legacy-watch index.py
+fi
+

example_config/cardmaker.yaml

@@ -1,3 +1,13 @@
 server:
   enable: True
   loglevel: "info"
+
+version:
+  0:
+    ongeki: 1.30.01
+    chuni: 2.00.00
+    maimai: 1.20.00
+  1:
+    ongeki: 1.35.03
+    chuni: 2.10.00
+    maimai: 1.30.00

example_config/chuni.yaml

@@ -15,6 +15,9 @@ version:
   12:
     rom: 2.05.00
     data: 2.05.00
+  13:
+    rom: 2.10.00
+    data: 2.10.00
 
 crypto:
   encrypted_only: False

example_config/core.yaml

@@ -4,6 +4,7 @@ server:
   allow_unregistered_serials: True
   name: "ARTEMiS"
   is_develop: True
+  threading: False
   log_dir: "logs"
 
 title:
@@ -32,6 +33,7 @@ allnet:
   loglevel: "info"
   port: 80
   allow_online_updates: False
+  update_cfg_folder: ""
 
 billing:
   port: 8443

example_config/idz.yaml

@@ -0,0 +1,11 @@
+server:
+  enable: True
+  loglevel: "info"
+  hostname: ""
+  news: ""
+  aes_key: ""
+
+ports:
+  userdb: 10000
+  match: 10010
+  echo: 10020

example_config/mai2.yaml

@@ -1,3 +1,8 @@
 server:
   enable: True
   loglevel: "info"
+
+deliver:
+  enable: False
+  udbdl_enable: False
+  content_folder: ""

example_config/ongeki.yaml

@@ -29,3 +29,9 @@ gachas:
     - 1156
     - 1163
     - 1164
+
+version:
+  6:
+    card_maker: 1.30.01
+  7:
+    card_maker: 1.35.03

example_config/pokken.yaml

@@ -2,8 +2,11 @@ server:
   hostname: "localhost"
   enable: True
   loglevel: "info"
-  port: 9000
-  port_stun: 9001
-  port_turn: 9002
-  port_admission: 9003
   auto_register: True
+  enable_matching: False
+  stun_server_host: "stunserver.stunprotocol.org"
+  stun_server_port: 3478
+
+ports:
+  game: 9000
+  admission: 9001

example_config/sao.yaml

@@ -0,0 +1,6 @@
+server:
+  hostname: "localhost"
+  enable: True
+  loglevel: "info"
+  port: 9000
+  auto_register: True

example_config/wacca.yaml

@@ -29,5 +29,8 @@ gates:
     - 17
     - 18
     - 19
+    - 20
     - 21
     - 22
+    - 23
+    - 24

index.py

@@ -11,7 +11,7 @@ from twisted.web import server, resource
 from twisted.internet import reactor, endpoints
 from twisted.web.http import Request
 from routes import Mapper
-
+from threading import Thread
 
 class HttpDispatcher(resource.Resource):
     def __init__(self, cfg: CoreConfig, config_dir: str):
@@ -26,7 +26,23 @@ class HttpDispatcher(resource.Resource):
         self.title = TitleServlet(cfg, config_dir)
         self.mucha = MuchaServlet(cfg, config_dir)
 
+        self.map_get.connect(
+            "allnet_downloadorder_ini",
+            "/dl/ini/{file}",
+            controller="allnet",
+            action="handle_dlorder_ini",
+            conditions=dict(method=["GET"]),
+        )
+
         self.map_post.connect(
+            "allnet_downloadorder_report",
+            "/dl/report",
+            controller="allnet",
+            action="handle_dlorder_report",
+            conditions=dict(method=["POST"]),
+        )
+
+        self.map_get.connect(
             "allnet_ping",
             "/naomitest.html",
             controller="allnet",
@@ -47,6 +63,27 @@ class HttpDispatcher(resource.Resource):
             action="handle_dlorder",
             conditions=dict(method=["POST"]),
         )
+        self.map_post.connect(
+            "allnet_loaderstaterecorder",
+            "/sys/servlet/LoaderStateRecorder",
+            controller="allnet",
+            action="handle_loaderstaterecorder",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_post.connect(
+            "allnet_alive",
+            "/sys/servlet/Alive",
+            controller="allnet",
+            action="handle_alive",
+            conditions=dict(method=["POST"]),
+        )
+        self.map_get.connect(
+            "allnet_alive",
+            "/sys/servlet/Alive",
+            controller="allnet",
+            action="handle_alive",
+            conditions=dict(method=["GET"]),
+        )
         self.map_post.connect(
             "allnet_billing",
             "/request",
@@ -95,7 +132,6 @@ class HttpDispatcher(resource.Resource):
         )
 
     def render_GET(self, request: Request) -> bytes:
-        self.logger.debug(request.uri)
         test = self.map_get.match(request.uri.decode())
         client_ip = Utils.get_ip_addr(request)
 
@@ -145,9 +181,16 @@ class HttpDispatcher(resource.Resource):
 
         if type(ret) == str:
             return ret.encode()
-        elif type(ret) == bytes:
+        
+        elif type(ret) == bytes or type(ret) == tuple: # allow for bytes or tuple (data, response code) responses
             return ret
+        
+        elif ret is None:
+            self.logger.warn(f"None returned by controller for {request.uri.decode()} endpoint")
+            return b""
+        
         else:
+            self.logger.warn(f"Unknown data type returned by controller for {request.uri.decode()} endpoint")
             return b""
 
 
@@ -240,4 +283,7 @@ if __name__ == "__main__":
             server.Site(dispatcher)
         )
 
-    reactor.run()  # type: ignore
+    if cfg.server.threading:
+        Thread(target=reactor.run, args=(False,)).start()
+    else:
+        reactor.run()

read.py

@@ -135,8 +135,7 @@ if __name__ == "__main__":
 
     for dir, mod in titles.items():
         if args.series in mod.game_codes:
-            handler = mod.reader(config, args.version,
+            handler = mod.reader(config, args.version, bin_arg, opt_arg, args.extra)
-                                 bin_arg, opt_arg, args.extra)
             handler.read()
 
     logger.info("Done")

readme.md

@@ -2,30 +2,36 @@
 A network service emulator for games running SEGA'S ALL.NET service, and similar.
 
 # Supported games
-Games listed below have been tested and confirmed working. Only game versions older then the current one in active use in arcades (n-0) or current game versions older then a year (y-1) are supported.
+Games listed below have been tested and confirmed working. Only game versions older then the version currently active in arcades, or games versions that have not recieved a major update in over one year, are supported.
-+ Chunithm
-    + All versions up to New!! Plus
 
-+ Crossbeats Rev
++ CHUNITHM
+    + All versions up to SUN
+
++ crossbeats REV.
     + All versions + omnimix
 
 + maimai DX
-    + All versions up to Festival
+    + All versions up to FESTiVAL
 
-+ Hatsune Miku Arcade
++ Hatsune Miku: Project DIVA Arcade
     + All versions
 
 + Card Maker
-    + 1.34.xx
+    + 1.30
-    + 1.35.xx
+    + 1.35
 
-+ Ongeki
++ O.N.G.E.K.I.
-    + All versions up to Bright Memory
+    + All versions up to bright MEMORY
 
-+ Wacca
++ WACCA
     + Lily R
     + Reverse
 
++ POKKÉN TOURNAMENT
+    + Final Online
+
++ Sword Art Online Arcade (partial support)
+    + Final
 
 ## Requirements
 - python 3 (tested working with 3.9 and 3.10, other versions YMMV)

requirements.txt

@@ -16,3 +16,4 @@ Routes
 bcrypt
 jinja2
 protobuf
+autobahn

titles/chuni/__init__.py

@@ -7,4 +7,4 @@ index = ChuniServlet
 database = ChuniData
 reader = ChuniReader
 game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW]
-current_schema_version = 3
+current_schema_version = 4

titles/chuni/base.py

@@ -4,7 +4,7 @@ from datetime import datetime, timedelta
 from time import strftime
 
 import pytz
-from typing import Dict, Any
+from typing import Dict, Any, List
 
 from core.config import CoreConfig
 from titles.chuni.const import ChuniConstants
@@ -44,13 +44,15 @@ class ChuniBase:
             # check if a user already has some pogress and if not add the
             # login bonus entry
             user_login_bonus = self.data.item.get_login_bonus(
-                user_id, self.version, preset["id"]
+                user_id, self.version, preset["presetId"]
             )
             if user_login_bonus is None:
-                self.data.item.put_login_bonus(user_id, self.version, preset["id"])
+                self.data.item.put_login_bonus(
+                    user_id, self.version, preset["presetId"]
+                )
                 # yeah i'm lazy
                 user_login_bonus = self.data.item.get_login_bonus(
-                    user_id, self.version, preset["id"]
+                    user_id, self.version, preset["presetId"]
                 )
 
             # skip the login bonus entirely if its already finished
@@ -66,13 +68,13 @@ class ChuniBase:
                 last_update_date = datetime.now()
 
                 all_login_boni = self.data.static.get_login_bonus(
-                    self.version, preset["id"]
+                    self.version, preset["presetId"]
                 )
 
                 # skip the current bonus preset if no boni were found
                 if all_login_boni is None or len(all_login_boni) < 1:
                     self.logger.warn(
-                        f"No bonus entries found for bonus preset {preset['id']}"
+                        f"No bonus entries found for bonus preset {preset['presetId']}"
                     )
                     continue
 
@@ -83,14 +85,14 @@ class ChuniBase:
                 if bonus_count > max_needed_days:
                     # assume that all login preset ids under 3000 needs to be
                     # looped, like 30 and 40 are looped, 40 does not work?
-                    if preset["id"] < 3000:
+                    if preset["presetId"] < 3000:
                         bonus_count = 1
                     else:
                         is_finished = True
 
                 # grab the item for the corresponding day
                 login_item = self.data.static.get_login_bonus_by_required_days(
-                    self.version, preset["id"], bonus_count
+                    self.version, preset["presetId"], bonus_count
                 )
                 if login_item is not None:
                     # now add the present to the database so the
@@ -108,7 +110,7 @@ class ChuniBase:
                 self.data.item.put_login_bonus(
                     user_id,
                     self.version,
-                    preset["id"],
+                    preset["presetId"],
                     bonusCount=bonus_count,
                     lastUpdateDate=last_update_date,
                     isWatched=False,
@@ -156,12 +158,18 @@ class ChuniBase:
 
         event_list = []
         for evt_row in game_events:
-            tmp = {}
+            event_list.append(
-            tmp["id"] = evt_row["eventId"]
+                {
-            tmp["type"] = evt_row["type"]
+                    "id": evt_row["eventId"],
-            tmp["startDate"] = "2017-12-05 07:00:00.0"
+                    "type": evt_row["type"],
-            tmp["endDate"] = "2099-12-31 00:00:00.0"
+                    # actually use the startDate from the import so it
-            event_list.append(tmp)
+                    # properly shows all the events when new ones are imported
+                    "startDate": datetime.strftime(
+                        evt_row["startDate"], "%Y-%m-%d %H:%M:%S"
+                    ),
+                    "endDate": "2099-12-31 00:00:00",
+                }
+            )
 
         return {
             "type": data["type"],
@@ -228,29 +236,36 @@ class ChuniBase:
     def handle_get_user_character_api_request(self, data: Dict) -> Dict:
         characters = self.data.item.get_characters(data["userId"])
         if characters is None:
-            return {}
+            return {
-        next_idx = -1
+                "userId": data["userId"],
+                "length": 0,
+                "nextIndex": -1,
+                "userCharacterList": [],
+            }
 
-        characterList = []
+        character_list = []
-        for x in range(int(data["nextIndex"]), len(characters)):
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(characters)):
             tmp = characters[x]._asdict()
             tmp.pop("user")
             tmp.pop("id")
-            characterList.append(tmp)
+            character_list.append(tmp)
 
-            if len(characterList) >= int(data["maxCount"]):
+            if len(character_list) >= max_ct:
                 break
 
-        if len(characterList) >= int(data["maxCount"]) and len(characters) > int(
+        if len(characters) >= next_idx + max_ct:
-            data["maxCount"]
+            next_idx += max_ct
-        ) + int(data["nextIndex"]):
+        else:
-            next_idx = int(data["maxCount"]) + int(data["nextIndex"]) + 1
+            next_idx = -1
 
         return {
             "userId": data["userId"],
-            "length": len(characterList),
+            "length": len(character_list),
             "nextIndex": next_idx,
-            "userCharacterList": characterList,
+            "userCharacterList": character_list,
         }
 
     def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
@@ -292,8 +307,8 @@ class ChuniBase:
             if len(user_course_list) >= max_ct:
                 break
 
-        if len(user_course_list) >= max_ct:
+        if len(user_course_list) >= next_idx + max_ct:
-            next_idx = next_idx + max_ct
+            next_idx += max_ct
         else:
             next_idx = -1
 
@@ -347,12 +362,23 @@ class ChuniBase:
         }
 
     def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
+        user_fav_item_list = []
+
+        # still needs to be implemented on WebUI
+        # 1: Music, 3: Character
+        fav_list = self.data.item.get_all_favorites(
+            data["userId"], self.version, fav_kind=int(data["kind"])
+        )
+        if fav_list is not None:
+            for fav in fav_list:
+                user_fav_item_list.append({"id": fav["favId"]})
+
         return {
             "userId": data["userId"],
-            "length": 0,
+            "length": len(user_fav_item_list),
             "kind": data["kind"],
             "nextIndex": -1,
-            "userFavoriteItemList": [],
+            "userFavoriteItemList": user_fav_item_list,
         }
 
     def handle_get_user_favorite_music_api_request(self, data: Dict) -> Dict:
@@ -375,7 +401,7 @@ class ChuniBase:
                 "userItemList": [],
             }
 
-        items: list[Dict[str, Any]] = []
+        items: List[Dict[str, Any]] = []
         for i in range(next_idx, len(user_item_list)):
             tmp = user_item_list[i]._asdict()
             tmp.pop("user")
@@ -387,13 +413,13 @@ class ChuniBase:
         xout = kind * 10000000000 + next_idx + len(items)
 
         if len(items) < int(data["maxCount"]):
-            nextIndex = 0
+            next_idx = 0
         else:
-            nextIndex = xout
+            next_idx = xout
 
         return {
             "userId": data["userId"],
-            "nextIndex": nextIndex,
+            "nextIndex": next_idx,
             "itemKind": kind,
             "length": len(items),
             "userItemList": items,
@@ -452,6 +478,7 @@ class ChuniBase:
                 "nextIndex": -1,
                 "userMusicList": [],  # 240
             }
+
         song_list = []
         next_idx = int(data["nextIndex"])
         max_ct = int(data["maxCount"])
@@ -474,10 +501,10 @@ class ChuniBase:
             if len(song_list) >= max_ct:
                 break
 
-        if len(song_list) >= max_ct:
+        if len(song_list) >= next_idx + max_ct:
             next_idx += max_ct
         else:
-            next_idx = 0
+            next_idx = -1
 
         return {
             "userId": data["userId"],
@@ -623,12 +650,15 @@ class ChuniBase:
             self.data.profile.put_profile_data(
                 user_id, self.version, upsert["userData"][0]
             )
+
         if "userDataEx" in upsert:
             self.data.profile.put_profile_data_ex(
                 user_id, self.version, upsert["userDataEx"][0]
             )
+
         if "userGameOption" in upsert:
             self.data.profile.put_profile_option(user_id, upsert["userGameOption"][0])
+
         if "userGameOptionEx" in upsert:
             self.data.profile.put_profile_option_ex(
                 user_id, upsert["userGameOptionEx"][0]
@@ -672,6 +702,10 @@ class ChuniBase:
 
         if "userPlaylogList" in upsert:
             for playlog in upsert["userPlaylogList"]:
+                # convert the player names to utf-8
+                playlog["playedUserName1"] = self.read_wtf8(playlog["playedUserName1"])
+                playlog["playedUserName2"] = self.read_wtf8(playlog["playedUserName2"])
+                playlog["playedUserName3"] = self.read_wtf8(playlog["playedUserName3"])
                 self.data.score.put_playlog(user_id, playlog)
 
         if "userTeamPoint" in upsert: