cm: Added individual Card Maker version and maimai DX passes working

Reviewed-on: https://gitea.tendokyu.moe/Hay1tsme/artemis/pulls/20

.gitignore

@@ -158,5 +158,6 @@ cert/*
 !cert/server.pem
 config/*
 deliver/*
+*.gz
 
 dbdump-*.json

Dockerfile

@@ -0,0 +1,21 @@
+FROM python:3.9.15-slim-bullseye
+
+RUN apt update && apt install default-libmysqlclient-dev build-essential libtk nodejs npm -y
+
+WORKDIR /app
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+RUN npm i -g nodemon
+
+COPY entrypoint.sh entrypoint.sh
+RUN chmod +x entrypoint.sh
+
+COPY index.py index.py
+COPY dbutils.py dbutils.py
+ADD core core
+ADD titles titles
+ADD config config
+ADD log log
+ADD cert cert
+
+ENTRYPOINT [ "/app/entrypoint.sh" ]

changelog.md

@@ -0,0 +1,49 @@
+# Changelog
+Documenting updates to ARTEMiS, to be updated every time the master branch is pushed to.
+
+## 2023042300
+### Wacca
++ Time free now works properly
++ Fix reverse gate mission causing a fatal error
++ Other misc. fixes
++ Latest DB: 5
+
+### Pokken
++ Added preliminary support
+    + Nothing saves currently, but the game will boot and function properly.
+
+### Initial D Zero
++ Added preliminary support
+    + Nothing saves currently, but the game will boot and function for the most part.
+
+### Mai2
++ Added support for Festival
++ Lasted DB Version: 4
+
+### Ongeki
++ Misc fixes
++ Lasted DB Version: 4
+
+### Diva
++ Misc fixes
++ Lasted DB Version: 4
+
+### Chuni
++ Fix network encryption
++ Add `handle_remove_token_api_request` for event mode
+
+### Allnet
++ Added download order support
+    + It is up to the sysop to provide the INI file, and host the files.
+    + ONLY for use with cabs. It's not checked currently, which it's why it's default disabled
+    + YMMV, use at your own risk
++ When running develop mode, games that are not recognised will still be able to authenticate.
+
+### Database
++ Add autoupgrade command
+    + Invoke to automatically upgrade all schemas to their latest versions
+
++ `version` arg no longer required, leave it blank to update the game schema to latest if it isn't already
+
+### Misc
++ Update example nginx config file

core/allnet.py

@@ -10,6 +10,7 @@ from Crypto.PublicKey import RSA
 from Crypto.Hash import SHA
 from Crypto.Signature import PKCS1_v1_5
 from time import strptime
+from os import path
 
 from core.config import CoreConfig
 from core.utils import Utils
@@ -55,7 +56,7 @@ class AllnetServlet:
             self.logger.error("No games detected!")
 
         for _, mod in plugins.items():
-            if hasattr(mod.index, "get_allnet_info"):
+            if hasattr(mod, "index") and hasattr(mod.index, "get_allnet_info"):
                 for code in mod.game_codes:
                     enabled, uri, host = mod.index.get_allnet_info(
                         code, self.config, self.config_folder
@@ -106,9 +107,13 @@ class AllnetServlet:
                 return self.dict_to_http_form_string([vars(resp)])
 
             else:
-                self.logger.info(f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}")
+                self.logger.info(
+                    f"Allowed unknown game {req.game_id} v{req.ver} to authenticate from {request_ip} due to 'is_develop' being enabled. S/N: {req.serial}"
+                )
                 resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/{req.game_id}/{req.ver.replace('.', '')}/"
                 resp.host = f"{self.config.title.hostname}:{self.config.title.port}"
+                
+                self.logger.debug(f"Allnet response: {vars(resp)}")
                 return self.dict_to_http_form_string([vars(resp)])
 
         resp.uri, resp.host = self.uri_registry[req.game_id]
@@ -188,15 +193,51 @@ class AllnetServlet:
                 self.logger.error(e)
             return b""
 
-        self.logger.info(f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}")
+        self.logger.info(
+            f"DownloadOrder from {request_ip} -> {req.game_id} v{req.ver} serial {req.serial}"
+        )
         resp = AllnetDownloadOrderResponse()
 
-        if not self.config.allnet.allow_online_updates:
+        if (
+            not self.config.allnet.allow_online_updates
+            or not self.config.allnet.update_cfg_folder
+        ):
             return self.dict_to_http_form_string([vars(resp)])
 
-        else:  # TODO: Actual dlorder response
+        else:  # TODO: Keychip check
+            if path.exists(
+                f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
+            ):
+                resp.uri = f"http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-app.ini"
+
+            if path.exists(
+                f"{self.config.allnet.update_cfg_folder}/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
+            ):
+                resp.uri += f"|http://{self.config.title.hostname}:{self.config.title.port}/dl/ini/{req.game_id}-{req.ver.replace('.', '')}-opt.ini"
+
+            self.logger.debug(f"Sending download uri {resp.uri}")
             return self.dict_to_http_form_string([vars(resp)])
 
+    def handle_dlorder_ini(self, request: Request, match: Dict) -> bytes:
+        if "file" not in match:
+            return b""
+
+        req_file = match["file"].replace("%0A", "")
+
+        if path.exists(f"{self.config.allnet.update_cfg_folder}/{req_file}"):
+            return open(
+                f"{self.config.allnet.update_cfg_folder}/{req_file}", "rb"
+            ).read()
+
+        self.logger.info(f"DL INI File {req_file} not found")
+        return b""
+
+    def handle_dlorder_report(self, request: Request, match: Dict) -> bytes:
+        self.logger.info(
+            f"DLI Report from {Utils.get_ip_addr(request)}: {request.content.getvalue()}"
+        )
+        return b""
+
     def handle_billing_request(self, request: Request, _: Dict):
         req_dict = self.billing_req_to_dict(request.content.getvalue())
         request_ip = Utils.get_ip_addr(request)
@@ -210,14 +251,18 @@ class AllnetServlet:
         signer = PKCS1_v1_5.new(rsa)
         digest = SHA.new()
 
-        kc_playlimit = int(req_dict[0]["playlimit"])
+        try:
-        kc_nearfull = int(req_dict[0]["nearfull"])
+            kc_playlimit = int(req_dict[0]["playlimit"])
-        kc_billigtype = int(req_dict[0]["billingtype"])
+            kc_nearfull = int(req_dict[0]["nearfull"])
-        kc_playcount = int(req_dict[0]["playcnt"])
+            kc_billigtype = int(req_dict[0]["billingtype"])
-        kc_serial: str = req_dict[0]["keychipid"]
+            kc_playcount = int(req_dict[0]["playcnt"])
-        kc_game: str = req_dict[0]["gameid"]
+            kc_serial: str = req_dict[0]["keychipid"]
-        kc_date = strptime(req_dict[0]["date"], "%Y%m%d%H%M%S")
+            kc_game: str = req_dict[0]["gameid"]
-        kc_serial_bytes = kc_serial.encode()
+            kc_date = strptime(req_dict[0]["date"], "%Y%m%d%H%M%S")
+            kc_serial_bytes = kc_serial.encode()
+        
+        except KeyError as e:
+            return f"result=5&linelimit=&message={e} field is missing".encode()
 
         machine = self.data.arcade.get_machine(kc_serial)
         if machine is None and not self.config.server.allow_unregistered_serials:
@@ -367,8 +412,8 @@ class AllnetPowerOnResponse3:
         self.uri = ""
         self.host = ""
         self.place_id = "123"
-        self.name = ""
+        self.name = "ARTEMiS"
-        self.nickname = ""
+        self.nickname = "ARTEMiS"
         self.region0 = "1"
         self.region_name0 = "W"
         self.region_name1 = ""
@@ -391,8 +436,8 @@ class AllnetPowerOnResponse2:
         self.uri = ""
         self.host = ""
         self.place_id = "123"
-        self.name = "Test"
+        self.name = "ARTEMiS"
-        self.nickname = "Test123"
+        self.nickname = "ARTEMiS"
         self.region0 = "1"
         self.region_name0 = "W"
         self.region_name1 = "X"
@@ -419,7 +464,7 @@ class AllnetDownloadOrderRequest:
 
 
 class AllnetDownloadOrderResponse:
-    def __init__(self, stat: int = 1, serial: str = "", uri: str = "null") -> None:
+    def __init__(self, stat: int = 1, serial: str = "", uri: str = "") -> None:
         self.stat = stat
         self.serial = serial
         self.uri = uri

core/config.py

@@ -188,6 +188,12 @@ class AllnetConfig:
             self.__config, "core", "allnet", "allow_online_updates", default=False
         )
 
+    @property
+    def update_cfg_folder(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "core", "allnet", "update_cfg_folder", default=""
+        )
+
 
 class BillingConfig:
     def __init__(self, parent_config: "CoreConfig") -> None:

core/data/database.py

@@ -1,5 +1,5 @@
 import logging, coloredlogs
-from typing import Optional
+from typing import Optional, Dict, List
 from sqlalchemy.orm import scoped_session, sessionmaker
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy import create_engine
@@ -32,7 +32,7 @@ class Data:
         self.arcade = ArcadeData(self.config, self.session)
         self.card = CardData(self.config, self.session)
         self.base = BaseData(self.config, self.session)
-        self.schema_ver_latest = 4
+        self.current_schema_version = 4
 
         log_fmt_str = "[%(asctime)s] %(levelname)s | Database | %(message)s"
         log_fmt = logging.Formatter(log_fmt_str)
@@ -71,7 +71,9 @@ class Data:
         games = Utils.get_all_titles()
         for game_dir, game_mod in games.items():
             try:
-                if hasattr(game_mod, "database") and hasattr(game_mod, "current_schema_version"):
+                if hasattr(game_mod, "database") and hasattr(
+                    game_mod, "current_schema_version"
+                ):
                     game_mod.database(self.config)
                 metadata.create_all(self.__engine.connect())
 
@@ -84,8 +86,8 @@ class Data:
                     f"Could not load database schema from {game_dir} - {e}"
                 )
 
-        self.logger.info(f"Setting base_schema_ver to {self.schema_ver_latest}")
+        self.logger.info(f"Setting base_schema_ver to {self.current_schema_version}")
-        self.base.set_schema_ver(self.schema_ver_latest)
+        self.base.set_schema_ver(self.current_schema_version)
 
         self.logger.info(
             f"Setting user auto_incrememnt to {self.config.database.user_table_autoincrement_start}"
@@ -129,9 +131,32 @@ class Data:
 
         self.create_database()
 
-    def migrate_database(self, game: str, version: int, action: str) -> None:
+    def migrate_database(self, game: str, version: Optional[int], action: str) -> None:
         old_ver = self.base.get_schema_ver(game)
         sql = ""
+        if version is None:
+            if not game == "CORE":
+                titles = Utils.get_all_titles()
+
+                for folder, mod in titles.items():
+                    if not mod.game_codes[0] == game:
+                        continue
+
+                    if hasattr(mod, "current_schema_version"):
+                        version = mod.current_schema_version
+
+                    else:
+                        self.logger.warn(
+                            f"current_schema_version not found for {folder}"
+                        )
+
+            else:
+                version = self.current_schema_version
+
+        if version is None:
+            self.logger.warn(
+                f"Could not determine latest version for {game}, please specify --version"
+            )
 
         if old_ver is None:
             self.logger.error(
@@ -263,17 +288,48 @@ class Data:
                 self.user.delete_user(user["id"])
 
     def autoupgrade(self) -> None:
-        all_games = self.base.get_all_schema_vers()
+        all_game_versions = self.base.get_all_schema_vers()
-        if all_games is None:
+        if all_game_versions is None:
             self.logger.warn("Failed to get schema versions")
+            return
 
-        for x in all_games:
+        all_games = Utils.get_all_titles()
+        all_games_list: Dict[str, int] = {}
+        for _, mod in all_games.items():
+            if hasattr(mod, "current_schema_version"):
+                all_games_list[mod.game_codes[0]] = mod.current_schema_version
+
+        for x in all_game_versions:
+            failed = False
             game = x["game"].upper()
-            update_ver = 1
+            update_ver = int(x["version"])
-            for y in range(2, 100):
+            latest_ver = all_games_list.get(game, 1)
-                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
+            if game == "CORE":
-                    update_ver = y
+                latest_ver = self.current_schema_version
-                else:
-                    break
 
-            self.migrate_database(game, update_ver, "upgrade")
+            if update_ver == latest_ver:
+                self.logger.info(f"{game} is already latest version")
+                continue
+
+            for y in range(update_ver + 1, latest_ver + 1):
+                if os.path.exists(f"core/data/schema/versions/{game}_{y}_upgrade.sql"):
+                    with open(
+                        f"core/data/schema/versions/{game}_{y}_upgrade.sql",
+                        "r",
+                        encoding="utf-8",
+                    ) as f:
+                        sql = f.read()
+
+                    result = self.base.execute(sql)
+                    if result is None:
+                        self.logger.error(
+                            f"Error execuing sql script for game {game} v{y}!"
+                        )
+                        failed = True
+                        break
+                else:
+                    self.logger.warning(f"Could not find script {game}_{y}_upgrade.sql")
+                    failed = True
+
+            if not failed:
+                self.base.set_schema_ver(latest_ver, game)

core/data/schema/base.py

@@ -47,7 +47,7 @@ class BaseData:
         res = None
 
         try:
-            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())} || {opts}")
+            self.logger.info(f"SQL Execute: {''.join(str(sql).splitlines())}")
             res = self.conn.execute(text(sql), opts)
 
         except SQLAlchemyError as e:

core/data/schema/user.py

@@ -80,6 +80,9 @@ class UserData(BaseData):
         if usr["password"] is None:
             return False
         
+        if passwd is None or not passwd:
+            return False
+
         return bcrypt.checkpw(passwd, usr["password"].encode())
 
     def reset_autoincrement(self, ai_value: int) -> None:

core/data/schema/versions/SBZV_4_rollback.sql

@@ -0,0 +1,9 @@
+ALTER TABLE diva_profile
+    DROP cnp_cid,
+    DROP cnp_val,
+    DROP cnp_rr,
+    DROP cnp_sp,
+    DROP btn_se_eqp,
+    DROP sld_se_eqp,
+    DROP chn_sld_se_eqp,
+    DROP sldr_tch_se_eqp;

core/data/schema/versions/SBZV_5_upgrade.sql

@@ -0,0 +1,9 @@
+ALTER TABLE diva_profile
+    ADD cnp_cid INT NOT NULL DEFAULT -1,
+    ADD cnp_val INT NOT NULL DEFAULT -1,
+    ADD cnp_rr INT NOT NULL DEFAULT -1,
+    ADD cnp_sp VARCHAR(255) NOT NULL DEFAULT "",
+    ADD btn_se_eqp INT NOT NULL DEFAULT -1,
+    ADD sld_se_eqp INT NOT NULL DEFAULT -1,
+    ADD chn_sld_se_eqp INT NOT NULL DEFAULT -1,
+    ADD sldr_tch_se_eqp INT NOT NULL DEFAULT -1;

core/data/schema/versions/SDBT_3_rollback.sql

@@ -0,0 +1,30 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    DROP COLUMN regionId,
+    DROP COLUMN machineType;
+
+ALTER TABLE chuni_static_events
+    DROP COLUMN startDate;
+
+ALTER TABLE chuni_profile_data
+    DROP COLUMN rankUpChallengeResults;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN presetId id INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD PRIMARY KEY(id);
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_uk UNIQUE(id, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY(presetId)
+    REFERENCES chuni_static_login_bonus_preset(id) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDBT_4_upgrade.sql

@@ -0,0 +1,29 @@
+SET FOREIGN_KEY_CHECKS = 0;
+
+ALTER TABLE chuni_score_playlog
+    ADD COLUMN regionId INT,
+    ADD COLUMN machineType INT;
+
+ALTER TABLE chuni_static_events
+    ADD COLUMN startDate TIMESTAMP NOT NULL DEFAULT current_timestamp();
+
+ALTER TABLE chuni_profile_data
+    ADD COLUMN rankUpChallengeResults JSON;
+
+ALTER TABLE chuni_static_login_bonus
+    DROP FOREIGN KEY chuni_static_login_bonus_ibfk_1;
+
+ALTER TABLE chuni_static_login_bonus_preset
+    CHANGE COLUMN id presetId INT NOT NULL;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP PRIMARY KEY;
+ALTER TABLE chuni_static_login_bonus_preset
+    DROP INDEX chuni_static_login_bonus_preset_uk;
+ALTER TABLE chuni_static_login_bonus_preset
+    ADD CONSTRAINT chuni_static_login_bonus_preset_pk PRIMARY KEY (presetId, version);
+
+ALTER TABLE chuni_static_login_bonus
+    ADD CONSTRAINT chuni_static_login_bonus_ibfk_1 FOREIGN KEY (presetId, version)
+    REFERENCES chuni_static_login_bonus_preset(presetId, version) ON UPDATE CASCADE ON DELETE CASCADE;
+
+SET FOREIGN_KEY_CHECKS = 1;

core/data/schema/versions/SDEZ_4_rollback.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP DEFAULT "2018-01-01 00:00:00.0",
+    CHANGE COLUMN endDate endDate TIMESTAMP DEFAULT "2038-01-01 00:00:00.0";

core/data/schema/versions/SDEZ_5_upgrade.sql

@@ -0,0 +1,3 @@
+ALTER TABLE mai2_item_card 
+    CHANGE COLUMN startDate startDate TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CHANGE COLUMN endDate endDate TIMESTAMP NOT NULL;

core/data/schema/versions/SDFE_3_rollback.sql

@@ -0,0 +1,2 @@
+SET FOREIGN_KEY_CHECKS=0;
+SET FOREIGN_KEY_CHECKS=1;

core/data/schema/versions/SDFE_4_rollback.sql

@@ -0,0 +1 @@
+ALTER TABLE wacca_profile DROP COLUMN playcount_time_free;

core/data/schema/versions/SDFE_4_upgrade.sql

@@ -0,0 +1 @@
+DELETE FROM wacca_item WHERE type=17 AND item_id=312002;

core/data/schema/versions/SDFE_5_upgrade.sql

@@ -0,0 +1 @@
+ALTER TABLE wacca_profile ADD playcount_time_free int(11) DEFAULT 0 NULL AFTER playcount_stageup;

core/frontend.py

@@ -71,8 +71,11 @@ class FrontendServlet(resource.Resource):
                     game_fe = game_mod.frontend(cfg, self.environment, config_dir)
                     self.game_list.append({"url": game_dir, "name": game_fe.nav_name})
                     fe_game.putChild(game_dir.encode(), game_fe)
-                except:
+
-                    raise
+                except Exception as e:
+                    self.logger.error(
+                        f"Failed to import frontend from {game_dir} because {e}"
+                    )
 
         self.environment.globals["game_list"] = self.game_list
         self.putChild(b"gate", FE_Gate(cfg, self.environment))
@@ -179,7 +182,7 @@ class FE_Gate(FE_Base):
             access_code: str = request.args[b"access_code"][0].decode()
             username: str = request.args[b"username"][0]
             email: str = request.args[b"email"][0].decode()
-            passwd: str = request.args[b"passwd"][0]
+            passwd: bytes = request.args[b"passwd"][0]
 
             uid = self.data.card.get_user_id_from_card(access_code)
             if uid is None:
@@ -194,7 +197,7 @@ class FE_Gate(FE_Base):
             if result is None:
                 return redirectTo(b"/gate?e=3", request)
 
-            if not self.data.user.check_password(uid, passwd.encode()):
+            if not self.data.user.check_password(uid, passwd):
                 return redirectTo(b"/gate", request)
 
             return redirectTo(b"/user", request)
@@ -225,8 +228,21 @@ class FE_User(FE_Base):
         if usr_sesh.userId == 0:
             return redirectTo(b"/gate", request)
         
+        cards = self.data.card.get_user_cards(usr_sesh.userId)
+        user = self.data.user.get_user(usr_sesh.userId)
+        card_data = []
+        for c in cards:
+            if c['is_locked']:
+                status = 'Locked'
+            elif c['is_banned']:
+                status = 'Banned'
+            else:
+                status = 'Active'
+            
+            card_data.append({'access_code': c['access_code'], 'status': status})
+
         return template.render(
-            title=f"{self.core_config.server.name} | Account", sesh=vars(usr_sesh)
+            title=f"{self.core_config.server.name} | Account", sesh=vars(usr_sesh), cards=card_data, username=user['username']
         ).encode("utf-16")
 
 

core/frontend/user/index.jinja

@@ -1,4 +1,31 @@
 {% extends "core/frontend/index.jinja" %}
 {% block content %}
-<h1>testing</h1>
+<h1>Management for {{ username }}</h1>
+<h2>Cards <button class="btn btn-success" data-bs-toggle="modal" data-bs-target="#card_add">Add</button></h2>
+<ul>
+{% for c in cards %}
+<li>{{ c.access_code }}: {{ c.status }} <button class="btn-danger btn">Delete</button></li>
+{% endfor %}
+</ul>
+<div class="modal fade" id="card_add" tabindex="-1" aria-labelledby="card_add_label" aria-hidden="true">
+  <div class="modal-dialog">
+    <div class="modal-content">
+      <div class="modal-header">
+        <h1 class="modal-title fs-5" id="card_add_label">Add Card</h1>
+        <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
+      </div>
+      <div class="modal-body">
+        HOW TO:<br>
+        Scan your card on any networked game and press the "View Access Code" button (varies by game) and enter the 20 digit code below.<br>
+        !!FOR AMUSEIC CARDS: DO NOT ENTER THE CODE SHOWN ON THE BACK OF THE CARD ITSELF OR IT WILL NOT WORK!!
+        <p /><label for="card_add_frm_access_code">Access Code:&nbsp;</label><input id="card_add_frm_access_code" maxlength="20" type="text" required> 
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-primary">Add</button>
+        <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+      </div>
+    </div>
+  </div>
+</div>
+
 {% endblock content %}

core/frontend/widgets/topbar.jinja

@@ -4,7 +4,7 @@
 <div style="background: #333; color: #f9f9f9; width: 80%; height: 50px; line-height: 50px; padding-left: 10px; float: left;">
     <a href=/><button class="btn btn-primary">Home</button></a>&nbsp;
     {% for game in game_list %}    
-    <a href=game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
+    <a href=/game/{{ game.url }}><button class="btn btn-success">{{ game.name }}</button></a>&nbsp;
     {% endfor %}
 </div>
 </div>

core/mucha.py

@@ -46,9 +46,7 @@ class MuchaServlet:
                 if enabled:
                     self.mucha_registry.append(game_cd)
 
-        self.logger.info(
+        self.logger.info(f"Serving {len(self.mucha_registry)} games")
-            f"Serving {len(self.mucha_registry)} games"
-        )
 
     def handle_boardauth(self, request: Request, _: Dict) -> bytes:
         req_dict = self.mucha_preprocess(request.content.getvalue())
@@ -62,9 +60,7 @@ class MuchaServlet:
 
         req = MuchaAuthRequest(req_dict)
         self.logger.debug(f"Mucha request {vars(req)}")
-        self.logger.info(
+        self.logger.info(f"Boardauth request from {client_ip} for {req.gameVer}")
-            f"Boardauth request from {client_ip} for {req.gameVer}"
-        )
 
         if req.gameCd not in self.mucha_registry:
             self.logger.warn(f"Unknown gameCd {req.gameCd}")
@@ -92,9 +88,7 @@ class MuchaServlet:
 
         req = MuchaUpdateRequest(req_dict)
         self.logger.debug(f"Mucha request {vars(req)}")
-        self.logger.info(
+        self.logger.info(f"Updatecheck request from {client_ip} for {req.gameVer}")
-            f"Updatecheck request from {client_ip} for {req.gameVer}"
-        )
 
         if req.gameCd not in self.mucha_registry:
             self.logger.warn(f"Unknown gameCd {req.gameCd}")

core/utils.py

@@ -16,7 +16,10 @@ class Utils:
                 if not dir.startswith("__"):
                     try:
                         mod = importlib.import_module(f"titles.{dir}")
-                        ret[dir] = mod
+                        if hasattr(mod, "game_codes") and hasattr(
+                            mod, "index"
+                        ):  # Minimum required to function
+                            ret[dir] = mod
 
                     except ImportError as e:
                         logging.getLogger("core").error(f"get_all_titles: {dir} - {e}")
@@ -25,4 +28,8 @@ class Utils:
 
     @classmethod
     def get_ip_addr(cls, req: Request) -> str:
-        return req.getAllHeaders()[b"x-forwarded-for"].decode() if b"x-forwarded-for" in req.getAllHeaders() else req.getClientAddress().host
+        return (
+            req.getAllHeaders()[b"x-forwarded-for"].decode()
+            if b"x-forwarded-for" in req.getAllHeaders()
+            else req.getClientAddress().host
+        )

dbutils.py

@@ -1,5 +1,6 @@
 import yaml
 import argparse
+import logging
 from core.config import CoreConfig
 from core.data import Data
 from os import path, mkdir, access, W_OK
@@ -32,7 +33,9 @@ if __name__ == "__main__":
 
     cfg = CoreConfig()
     if path.exists(f"{args.config}/core.yaml"):
-        cfg.update(yaml.safe_load(open(f"{args.config}/core.yaml")))
+        cfg_dict = yaml.safe_load(open(f"{args.config}/core.yaml"))
+        cfg_dict.get("database", {})["loglevel"] = "info"
+        cfg.update(cfg_dict)
 
     if not path.exists(cfg.server.log_dir):
         mkdir(cfg.server.log_dir)
@@ -45,7 +48,6 @@ if __name__ == "__main__":
 
     data = Data(cfg)
 
-
     if args.action == "create":
         data.create_database()
 
@@ -54,15 +56,22 @@ if __name__ == "__main__":
 
     elif args.action == "upgrade" or args.action == "rollback":
         if args.version is None:
-            data.logger.error("Must set game and version to migrate to")
+            data.logger.warn("No version set, upgrading to latest")
-            exit(0)
 
         if args.game is None:
-            data.logger.info("No game set, upgrading core schema")
+            data.logger.warn("No game set, upgrading core schema")
-            data.migrate_database("CORE", int(args.version), args.action)
+            data.migrate_database(
+                "CORE",
+                int(args.version) if args.version is not None else None,
+                args.action,
+            )
 
         else:
-            data.migrate_database(args.game, int(args.version), args.action)
+            data.migrate_database(
+                args.game,
+                int(args.version) if args.version is not None else None,
+                args.action,
+            )
 
     elif args.action == "autoupgrade":
         data.autoupgrade()

docker-compose.yml

@@ -0,0 +1,57 @@
+version: "3.9"
+services:
+  app:
+    hostname: ma.app
+    build: .
+    volumes:
+      - ./aime:/app/aime
+
+    environment:
+      CFG_DEV: 1
+      CFG_CORE_SERVER_HOSTNAME: 0.0.0.0
+      CFG_CORE_DATABASE_HOST: ma.db
+      CFG_CORE_MEMCACHED_HOSTNAME: ma.memcached
+      CFG_CORE_AIMEDB_KEY: keyhere
+      CFG_CHUNI_SERVER_LOGLEVEL: debug
+
+    ports:
+    - "80:80"
+    - "8443:8443"
+    - "22345:22345"
+
+    - "8080:8080"
+    - "8090:8090"
+
+    depends_on:
+      db:
+        condition: service_healthy
+
+  db:
+    hostname: ma.db
+    image: mysql:8.0.31-debian
+    environment:
+      MYSQL_DATABASE: aime
+      MYSQL_USER: aime
+      MYSQL_PASSWORD: aime
+      MYSQL_ROOT_PASSWORD: AimeRootPassword
+    healthcheck:
+      test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"]
+      timeout: 5s
+      retries: 5
+
+
+  memcached:
+    hostname: ma.memcached
+    image: memcached:1.6.17-bullseye
+
+  phpmyadmin:
+    hostname: ma.phpmyadmin
+    image: phpmyadmin:latest
+    environment:
+      PMA_HOSTS: ma.db
+      PMA_USER: root
+      PMA_PASSWORD: AimeRootPassword
+      APACHE_PORT: 8080
+    ports:
+    - "8080:8080"
+

docs/game_specific_info.md

@@ -9,42 +9,44 @@ using the megaime database. Clean installations always create the latest databas
 # Table of content
 
 - [Supported Games](#supported-games)
-    - [Chunithm](#chunithm)
+    - [CHUNITHM](#chunithm)
     - [crossbeats REV.](#crossbeats-rev)
     - [maimai DX](#maimai-dx)
     - [O.N.G.E.K.I.](#o-n-g-e-k-i)
     - [Card Maker](#card-maker)
     - [WACCA](#wacca)
+    - [Sword Art Online Arcade](#sao)
 
 
 # Supported Games
 
 Games listed below have been tested and confirmed working.
 
-## Chunithm
+## CHUNITHM
 
 ### SDBT
 
-| Version ID | Version Name       |
+| Version ID | Version Name          |
-|------------|--------------------|
+|------------|-----------------------|
-| 0          | Chunithm           |
+| 0          | CHUNITHM              |
-| 1          | Chunithm+          |
+| 1          | CHUNITHM PLUS         |
-| 2          | Chunithm Air       |
+| 2          | CHUNITHM AIR          |
-| 3          | Chunithm Air +     |
+| 3          | CHUNITHM AIR PLUS     |
-| 4          | Chunithm Star      |
+| 4          | CHUNITHM STAR         |
-| 5          | Chunithm Star +    |
+| 5          | CHUNITHM STAR PLUS    |
-| 6          | Chunithm Amazon    |
+| 6          | CHUNITHM AMAZON       |
-| 7          | Chunithm Amazon +  |
+| 7          | CHUNITHM AMAZON PLUS  |
-| 8          | Chunithm Crystal   |
+| 8          | CHUNITHM CRYSTAL      |
-| 9          | Chunithm Crystal + |
+| 9          | CHUNITHM CRYSTAL PLUS |
-| 10         | Chunithm Paradise  |
+| 10         | CHUNITHM PARADISE     |
 
 ### SDHD/SDBT
 
-| Version ID | Version Name    |
+| Version ID | Version Name        |
-|------------|-----------------|
+|------------|---------------------|
-| 11         | Chunithm New!!  |
+| 11         | CHUNITHM NEW!!      |
-| 12         | Chunithm New!!+ |
+| 12         | CHUNITHM NEW PLUS!! |
+| 13         | CHUNITHM SUN        |
 
 
 ### Importer
@@ -60,14 +62,33 @@ The importer for Chunithm will import: Events, Music, Charge Items and Avatar Ac
 ### Database upgrade
 
 Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see
-which version is the latest, f.e. `SDBT_3_upgrade.sql`. In order to upgrade to version 3 in this case you need to
+which version is the latest, f.e. `SDBT_4_upgrade.sql`. In order to upgrade to version 4 in this case you need to
 perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDBT --version 2 upgrade
+python dbutils.py --game SDBT upgrade
-python dbutils.py --game SDBT --version 3 upgrade
 ```
 
+### Online Battle
+
+**Only matchmaking (with your imaginary friends) is supported! Online Battle does not (yet?) work!**
+
+The first person to start the Online Battle (now called host) will create a "matching room" with a given `roomId`, after that max 3 other people can join the created room.
+Non used slots during the matchmaking will be filled with CPUs after the timer runs out.
+As soon as a new member will join the room the timer will jump back to 60 secs again.
+Sending those 4 messages to all other users is also working properly.
+In order to use the Online Battle every user needs the same ICF, same rom version and same data version!
+If a room is full a new room will be created if another user starts an Online Battle.
+After a failed Online Battle the room will be deleted. The host is used for the timer countdown, so if the connection failes to the host the timer will stop and could create a "frozen" state.
+
+#### Information/Problems:
+
+- Online Battle uses UDP hole punching and opens port 50201?
+- `reflectorUri` seems related to that?
+- Timer countdown should be handled globally and not by one user
+- Game can freeze or can crash if someone (especially the host) leaves the matchmaking
+
+
 ## crossbeats REV.
 
 ### SDCA
@@ -112,9 +133,9 @@ Config file is located in `config/cxb.yaml`.
 | 1          | maimai DX PLUS          |
 | 2          | maimai DX Splash        |
 | 3          | maimai DX Splash PLUS   |
-| 4          | maimai DX Universe      |
+| 4          | maimai DX UNiVERSE      |
-| 5          | maimai DX Universe PLUS |
+| 5          | maimai DX UNiVERSE PLUS |
-| 6          | maimai DX Festival      |
+| 6          | maimai DX FESTiVAL      |
 
 ### Importer
 
@@ -134,7 +155,7 @@ crash without Events!**
 Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDEZ_2_upgrade.sql`. In order to upgrade to version 2 in this case you need to perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDEZ --version 2 upgrade
+python dbutils.py --game SDEZ upgrade
 ```
 
 ## Hatsune Miku Project Diva
@@ -175,9 +196,7 @@ which version is the latest, f.e. `SBZV_4_upgrade.sql`. In order to upgrade to v
 perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SBZV --version 2 upgrade
+python dbutils.py --game SBZV upgrade
-python dbutils.py --game SBZV --version 3 upgrade
-python dbutils.py --game SBZV --version 4 upgrade
 ```
 
 ## O.N.G.E.K.I.
@@ -225,9 +244,7 @@ which version is the latest, f.e. `SDDT_4_upgrade.sql`. In order to upgrade to v
 perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDDT --version 2 upgrade
+python dbutils.py --game SDDT upgrade
-python dbutils.py --game SDDT --version 3 upgrade
-python dbutils.py --game SDDT --version 4 upgrade
 ```
 
 ## Card Maker
@@ -236,20 +253,20 @@ python dbutils.py --game SDDT --version 4 upgrade
 
 | Version ID | Version Name    |
 |------------|-----------------|
-| 0          | Card Maker 1.34 |
+| 0          | Card Maker 1.30 |
 | 1          | Card Maker 1.35 |
 
 
 ### Support status
 
-* Card Maker 1.34:
+* Card Maker 1.30:
-  * Chunithm New!!: Yes
+  * CHUNITHM NEW!!: Yes
-  * maimai DX Universe: Yes
+  * maimai DX UNiVERSE: Yes
   * O.N.G.E.K.I. Bright: Yes
 
 * Card Maker 1.35:
-  * Chunithm New!!+: Yes
+  * CHUNITHM SUN: Yes (NEW PLUS!! up to A032)
-  * maimai DX Universe PLUS: Yes
+  * maimai DX FESTiVAL: Yes (up to A035) (UNiVERSE PLUS up to A031)
   * O.N.G.E.K.I. Bright Memory: Yes
 
 
@@ -268,19 +285,46 @@ python read.py --series SDED --version <version ID> --binfolder titles/cm/cm_dat
 python read.py --series SDDT --version <version ID> --binfolder /path/to/game/folder --optfolder /path/to/game/option/folder
 ```
 
-Also make sure to import all maimai and Chunithm data as well:
+Also make sure to import all maimai DX and CHUNITHM data as well:
 
 ```shell
 python read.py --series SDED --version <version ID> --binfolder /path/to/cardmaker/CardMaker_Data
 ```
 
-The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai/Chunithm) and the hardcoded
+The importer for Card Maker will import all required Gachas (Banners) and cards (for maimai DX/CHUNITHM) and the hardcoded
 Cards for each Gacha (O.N.G.E.K.I. only).
 
 **NOTE: Without executing the importer Card Maker WILL NOT work!**
 
 
-### O.N.G.E.K.I. Gachas
+### Config setup
+
+Make sure to update your `config/cardmaker.yaml` with the correct version for each game. To get the current version required to run a specific game, open every opt (Axxx) folder descending until you find all three folders:
+
+- `MU3`: O.N.G.E.K.I.
+- `MAI`: maimai DX
+- `CHU`: CHUNITHM
+
+Inside each folder is a `DataConfig.xml` file, for example:
+
+`MU3/DataConfig.xml`:
+```xml
+  <cardMakerVersion>
+    <major>1</major>
+    <minor>35</minor>
+    <release>3</release>
+  </cardMakerVersion>
+```
+
+Now update your `config/cardmaker.yaml` with the correct version number, for example:
+
+```yaml
+version:
+  1: # Card Maker 1.35
+    ongeki: 1.35.03
+```	 
+
+### O.N.G.E.K.I.
 
 Gacha "無料ガチャ" can only pull from the free cards with the following probabilities: 94%: R, 5% SR and 1% chance of
 getting an SSR card
@@ -293,20 +337,24 @@ and 3% chance of getting an SSR card
 All other (limited) gachas can pull from every card added to ongeki_static_cards but with the promoted cards
 (click on the green button under the banner) having a 10 times higher chance to get pulled
 
-### Chunithm Gachas
+### CHUNITHM
 
-All cards in Chunithm (basically just the characters) have the same rarity to it just pulls randomly from all cards
+All cards in CHUNITHM (basically just the characters) have the same rarity to it just pulls randomly from all cards
 from a given gacha but made sure you cannot pull the same card twice in the same 5 times gacha roll.
 
+### maimai DX
+
+Printed maimai DX cards: Freedom (`cardTypeId=6`) or Gold Pass (`cardTypeId=4`) can now be selected during the login process. You can only have ONE Freedom and ONE Gold Pass active at a given time. The cards will expire after 15 days.
+
+Thanks GetzeAvenue for the `selectedCardList` rarity hint!
+
 ### Notes
 
-Card Maker 1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35 will only load an O.N.G.E.K.I.
+Card Maker 1.30-1.34 will only load an O.N.G.E.K.I. Bright profile (1.30). Card Maker 1.35+ will only load an O.N.G.E.K.I.
 Bright Memory profile (1.35).
-The gachas inside the `ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
+The gachas inside the `config/ongeki.yaml` will make sure only the right gacha ids for the right CM version will be loaded.
 Gacha IDs up to 1140 will be loaded for CM 1.34 and all gachas will be loaded for CM 1.35.
 
-**NOTE: There is currently no way to load/use the (printed) maimai DX cards!**
-
 ## WACCA
 
 ### SDFE
@@ -347,6 +395,49 @@ Config file is located in `config/wacca.yaml`.
 Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDFE_3_upgrade.sql`. In order to upgrade to version 3 in this case you need to perform all previous updates as well:
 
 ```shell
-python dbutils.py --game SDFE --version 2 upgrade
+python dbutils.py --game SDFE upgrade
-python dbutils.py --game SDFE --version 3 upgrade
 ```
+
+## SAO
+
+### SDEW
+
+| Version ID | Version Name  |
+|------------|---------------|
+| 0          | SAO           |
+
+
+### Importer
+
+In order to use the importer locate your game installation folder and execute:
+
+```shell
+python read.py --series SDEW --version <version ID> --binfolder /path/to/game/extractedassets
+```
+
+The importer for SAO will import all items, heroes, support skills and titles data.
+
+### Config
+
+Config file is located in `config/sao.yaml`.
+
+| Option             | Info                                                                        |
+|--------------------|-----------------------------------------------------------------------------|
+| `hostname`         | Changes the server listening address for Mucha                              |
+| `port`             | Changes the listing port                                                    |
+| `auto_register`    | Allows the game to handle the automatic registration of new cards           |
+
+
+### Database upgrade
+
+Always make sure your database (tables) are up-to-date, to do so go to the `core/data/schema/versions` folder and see which version is the latest, f.e. `SDEW_1_upgrade.sql`. In order to upgrade to version 3 in this case you need to perform all previous updates as well:
+
+```shell
+python dbutils.py --game SDEW upgrade
+```
+
+### Credits for SAO support:
+
+- Midorica - Limited Network Support
+- Dniel97 - Helping with network base
+- tungnotpunk - Source

entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [[ -z "${CFG_DEV}" ]]; then
+    echo Production mode
+    python3 index.py
+else
+    echo Development mode
+    python3 dbutils.py create
+    nodemon -w aime --legacy-watch index.py
+fi
+

example_config/cardmaker.yaml

@@ -1,3 +1,13 @@
 server:
   enable: True
   loglevel: "info"
+
+version:
+  0:
+    ongeki: 1.30.01
+    chuni: 2.00.00
+    maimai: 1.20.00
+  1:
+    ongeki: 1.35.03
+    chuni: 2.10.00
+    maimai: 1.30.00

example_config/chuni.yaml

@@ -15,6 +15,9 @@ version:
   12:
     rom: 2.05.00
     data: 2.05.00
+  13:
+    rom: 2.10.00
+    data: 2.10.00
 
 crypto:
   encrypted_only: False

example_config/core.yaml

@@ -32,6 +32,7 @@ allnet:
   loglevel: "info"
   port: 80
   allow_online_updates: False
+  update_cfg_folder: ""
 
 billing:
   port: 8443

example_config/idz.yaml

@@ -0,0 +1,11 @@
+server:
+  enable: True
+  loglevel: "info"
+  hostname: ""
+  news: ""
+  aes_key: ""
+
+ports:
+  userdb: 10000
+  match: 10010
+  echo: 10020

example_config/sao.yaml

@@ -0,0 +1,6 @@
+server:
+  hostname: "localhost"
+  enable: True
+  loglevel: "info"
+  port: 9000
+  auto_register: True

example_config/wacca.yaml

@@ -29,5 +29,8 @@ gates:
     - 17
     - 18
     - 19
+    - 20
     - 21
     - 22
+    - 23
+    - 24

index.py

@@ -26,7 +26,23 @@ class HttpDispatcher(resource.Resource):
         self.title = TitleServlet(cfg, config_dir)
         self.mucha = MuchaServlet(cfg, config_dir)
 
+        self.map_get.connect(
+            "allnet_downloadorder_ini",
+            "/dl/ini/{file}",
+            controller="allnet",
+            action="handle_dlorder_ini",
+            conditions=dict(method=["GET"]),
+        )
+
         self.map_post.connect(
+            "allnet_downloadorder_report",
+            "/dl/report",
+            controller="allnet",
+            action="handle_dlorder_report",
+            conditions=dict(method=["POST"]),
+        )
+
+        self.map_get.connect(
             "allnet_ping",
             "/naomitest.html",
             controller="allnet",

read.py

@@ -135,8 +135,7 @@ if __name__ == "__main__":
 
     for dir, mod in titles.items():
         if args.series in mod.game_codes:
-            handler = mod.reader(config, args.version,
+            handler = mod.reader(config, args.version, bin_arg, opt_arg, args.extra)
-                                 bin_arg, opt_arg, args.extra)
             handler.read()
 
     logger.info("Done")

readme.md

@@ -2,30 +2,33 @@
 A network service emulator for games running SEGA'S ALL.NET service, and similar.
 
 # Supported games
-Games listed below have been tested and confirmed working. Only game versions older then the current one in active use in arcades (n-0) or current game versions older then a year (y-1) are supported.
+Games listed below have been tested and confirmed working. Only game versions older then the version currently active in arcades, or games versions that have not recieved a major update in over one year, are supported.
-+ Chunithm
-    + All versions up to New!! Plus
 
-+ Crossbeats Rev
++ CHUNITHM
+    + All versions up to SUN
+
++ crossbeats REV.
     + All versions + omnimix
 
 + maimai DX
-    + All versions up to Festival
+    + All versions up to FESTiVAL
 
-+ Hatsune Miku Arcade
++ Hatsune Miku: Project DIVA Arcade
     + All versions
 
 + Card Maker
-    + 1.34.xx
+    + 1.30
-    + 1.35.xx
+    + 1.35
 
-+ Ongeki
++ O.N.G.E.K.I.
     + All versions up to Bright Memory
 
-+ Wacca
++ WACCA
     + Lily R
     + Reverse
 
++ POKKÉN TOURNAMENT
+    + Final Online
 
 ## Requirements
 - python 3 (tested working with 3.9 and 3.10, other versions YMMV)

titles/chuni/__init__.py

@@ -7,4 +7,4 @@ index = ChuniServlet
 database = ChuniData
 reader = ChuniReader
 game_codes = [ChuniConstants.GAME_CODE, ChuniConstants.GAME_CODE_NEW]
-current_schema_version = 3
+current_schema_version = 4

titles/chuni/base.py

@@ -44,13 +44,15 @@ class ChuniBase:
             # check if a user already has some pogress and if not add the
             # login bonus entry
             user_login_bonus = self.data.item.get_login_bonus(
-                user_id, self.version, preset["id"]
+                user_id, self.version, preset["presetId"]
             )
             if user_login_bonus is None:
-                self.data.item.put_login_bonus(user_id, self.version, preset["id"])
+                self.data.item.put_login_bonus(
+                    user_id, self.version, preset["presetId"]
+                )
                 # yeah i'm lazy
                 user_login_bonus = self.data.item.get_login_bonus(
-                    user_id, self.version, preset["id"]
+                    user_id, self.version, preset["presetId"]
                 )
 
             # skip the login bonus entirely if its already finished
@@ -66,13 +68,13 @@ class ChuniBase:
                 last_update_date = datetime.now()
 
                 all_login_boni = self.data.static.get_login_bonus(
-                    self.version, preset["id"]
+                    self.version, preset["presetId"]
                 )
 
                 # skip the current bonus preset if no boni were found
                 if all_login_boni is None or len(all_login_boni) < 1:
                     self.logger.warn(
-                        f"No bonus entries found for bonus preset {preset['id']}"
+                        f"No bonus entries found for bonus preset {preset['presetId']}"
                     )
                     continue
 
@@ -83,14 +85,14 @@ class ChuniBase:
                 if bonus_count > max_needed_days:
                     # assume that all login preset ids under 3000 needs to be
                     # looped, like 30 and 40 are looped, 40 does not work?
-                    if preset["id"] < 3000:
+                    if preset["presetId"] < 3000:
                         bonus_count = 1
                     else:
                         is_finished = True
 
                 # grab the item for the corresponding day
                 login_item = self.data.static.get_login_bonus_by_required_days(
-                    self.version, preset["id"], bonus_count
+                    self.version, preset["presetId"], bonus_count
                 )
                 if login_item is not None:
                     # now add the present to the database so the
@@ -108,7 +110,7 @@ class ChuniBase:
                 self.data.item.put_login_bonus(
                     user_id,
                     self.version,
-                    preset["id"],
+                    preset["presetId"],
                     bonusCount=bonus_count,
                     lastUpdateDate=last_update_date,
                     isWatched=False,
@@ -156,12 +158,18 @@ class ChuniBase:
 
         event_list = []
         for evt_row in game_events:
-            tmp = {}
+            event_list.append(
-            tmp["id"] = evt_row["eventId"]
+                {
-            tmp["type"] = evt_row["type"]
+                    "id": evt_row["eventId"],
-            tmp["startDate"] = "2017-12-05 07:00:00.0"
+                    "type": evt_row["type"],
-            tmp["endDate"] = "2099-12-31 00:00:00.0"
+                    # actually use the startDate from the import so it
-            event_list.append(tmp)
+                    # properly shows all the events when new ones are imported
+                    "startDate": datetime.strftime(
+                        evt_row["startDate"], "%Y-%m-%d %H:%M:%S"
+                    ),
+                    "endDate": "2099-12-31 00:00:00",
+                }
+            )
 
         return {
             "type": data["type"],
@@ -228,29 +236,36 @@ class ChuniBase:
     def handle_get_user_character_api_request(self, data: Dict) -> Dict:
         characters = self.data.item.get_characters(data["userId"])
         if characters is None:
-            return {}
+            return {
-        next_idx = -1
+                "userId": data["userId"],
+                "length": 0,
+                "nextIndex": -1,
+                "userCharacterList": [],
+            }
 
-        characterList = []
+        character_list = []
-        for x in range(int(data["nextIndex"]), len(characters)):
+        next_idx = int(data["nextIndex"])
+        max_ct = int(data["maxCount"])
+
+        for x in range(next_idx, len(characters)):
             tmp = characters[x]._asdict()
             tmp.pop("user")
             tmp.pop("id")
-            characterList.append(tmp)
+            character_list.append(tmp)
 
-            if len(characterList) >= int(data["maxCount"]):
+            if len(character_list) >= max_ct:
                 break
 
-        if len(characterList) >= int(data["maxCount"]) and len(characters) > int(
+        if len(characters) >= next_idx + max_ct:
-            data["maxCount"]
+            next_idx += max_ct
-        ) + int(data["nextIndex"]):
+        else:
-            next_idx = int(data["maxCount"]) + int(data["nextIndex"]) + 1
+            next_idx = -1
 
         return {
             "userId": data["userId"],
-            "length": len(characterList),
+            "length": len(character_list),
             "nextIndex": next_idx,
-            "userCharacterList": characterList,
+            "userCharacterList": character_list,
         }
 
     def handle_get_user_charge_api_request(self, data: Dict) -> Dict:
@@ -292,8 +307,8 @@ class ChuniBase:
             if len(user_course_list) >= max_ct:
                 break
 
-        if len(user_course_list) >= max_ct:
+        if len(user_course_list) >= next_idx + max_ct:
-            next_idx = next_idx + max_ct
+            next_idx += max_ct
         else:
             next_idx = -1
 
@@ -347,12 +362,23 @@ class ChuniBase:
         }
 
     def handle_get_user_favorite_item_api_request(self, data: Dict) -> Dict:
+        user_fav_item_list = []
+
+        # still needs to be implemented on WebUI
+        # 1: Music, 3: Character
+        fav_list = self.data.item.get_all_favorites(
+            data["userId"], self.version, fav_kind=int(data["kind"])
+        )
+        if fav_list is not None:
+            for fav in fav_list:
+                user_fav_item_list.append({"id": fav["favId"]})
+
         return {
             "userId": data["userId"],
-            "length": 0,
+            "length": len(user_fav_item_list),
             "kind": data["kind"],
             "nextIndex": -1,
-            "userFavoriteItemList": [],
+            "userFavoriteItemList": user_fav_item_list,
         }
 
     def handle_get_user_favorite_music_api_request(self, data: Dict) -> Dict:
@@ -387,13 +413,13 @@ class ChuniBase:
         xout = kind * 10000000000 + next_idx + len(items)
 
         if len(items) < int(data["maxCount"]):
-            nextIndex = 0
+            next_idx = 0
         else:
-            nextIndex = xout
+            next_idx = xout
 
         return {
             "userId": data["userId"],
-            "nextIndex": nextIndex,
+            "nextIndex": next_idx,
             "itemKind": kind,
             "length": len(items),
             "userItemList": items,
@@ -452,6 +478,7 @@ class ChuniBase:
                 "nextIndex": -1,
                 "userMusicList": [],  # 240
             }
+
         song_list = []
         next_idx = int(data["nextIndex"])
         max_ct = int(data["maxCount"])
@@ -474,10 +501,10 @@ class ChuniBase:
             if len(song_list) >= max_ct:
                 break
 
-        if len(song_list) >= max_ct:
+        if len(song_list) >= next_idx + max_ct:
             next_idx += max_ct
         else:
-            next_idx = 0
+            next_idx = -1
 
         return {
             "userId": data["userId"],
@@ -623,12 +650,15 @@ class ChuniBase:
             self.data.profile.put_profile_data(
                 user_id, self.version, upsert["userData"][0]
             )
+
         if "userDataEx" in upsert:
             self.data.profile.put_profile_data_ex(
                 user_id, self.version, upsert["userDataEx"][0]
             )
+
         if "userGameOption" in upsert:
             self.data.profile.put_profile_option(user_id, upsert["userGameOption"][0])
+
         if "userGameOptionEx" in upsert:
             self.data.profile.put_profile_option_ex(
                 user_id, upsert["userGameOptionEx"][0]
@@ -672,6 +702,10 @@ class ChuniBase:
 
         if "userPlaylogList" in upsert:
             for playlog in upsert["userPlaylogList"]:
+                # convert the player names to utf-8
+                playlog["playedUserName1"] = self.read_wtf8(playlog["playedUserName1"])
+                playlog["playedUserName2"] = self.read_wtf8(playlog["playedUserName2"])
+                playlog["playedUserName3"] = self.read_wtf8(playlog["playedUserName3"])
                 self.data.score.put_playlog(user_id, playlog)
 
         if "userTeamPoint" in upsert:

titles/chuni/const.py

@@ -17,21 +17,23 @@ class ChuniConstants:
     VER_CHUNITHM_PARADISE = 10
     VER_CHUNITHM_NEW = 11
     VER_CHUNITHM_NEW_PLUS = 12
+    VER_CHUNITHM_SUN = 13
 
     VERSION_NAMES = [
-        "Chunithm",
+        "CHUNITHM",
-        "Chunithm+",
+        "CHUNITHM PLUS",
-        "Chunithm Air",
+        "CHUNITHM AIR",
-        "Chunithm Air+",
+        "CHUNITHM AIR PLUS",
-        "Chunithm Star",
+        "CHUNITHM STAR",
-        "Chunithm Star+",
+        "CHUNITHM STAR  PLUS",
-        "Chunithm Amazon",
+        "CHUNITHM AMAZON",
-        "Chunithm Amazon+",
+        "CHUNITHM AMAZON PLUS",
-        "Chunithm Crystal",
+        "CHUNITHM CRYSTAL",
-        "Chunithm Crystal+",
+        "CHUNITHM CRYSTAL PLUS",
-        "Chunithm Paradise",
+        "CHUNITHM PARADISE",
-        "Chunithm New!!",
+        "CHUNITHM NEW!!",
-        "Chunithm New!!+",
+        "CHUNITHM NEW PLUS!!",
+        "CHUNITHM SUN"
     ]
 
     @classmethod

titles/chuni/index.py

@@ -29,6 +29,7 @@ from titles.chuni.crystalplus import ChuniCrystalPlus
 from titles.chuni.paradise import ChuniParadise
 from titles.chuni.new import ChuniNew
 from titles.chuni.newplus import ChuniNewPlus
+from titles.chuni.sun import ChuniSun
 
 
 class ChuniServlet:
@@ -55,6 +56,7 @@ class ChuniServlet:
             ChuniParadise,
             ChuniNew,
             ChuniNewPlus,
+            ChuniSun,
         ]
 
         self.logger = logging.getLogger("chuni")
@@ -89,14 +91,29 @@ class ChuniServlet:
 
             self.hash_table[version] = {}
 
-            method_list = [method for method in dir(self.versions[version]) if not method.startswith('__')]
+            method_list = [
+                method
+                for method in dir(self.versions[version])
+                if not method.startswith("__")
+            ]
             for method in method_list:
                 method_fixed = inflection.camelize(method)[6:-7]
-                hash = PBKDF2(method_fixed, bytes.fromhex(keys[2]), 128, count=44, hmac_hash_module=SHA1)
+                # number of iterations was changed to 70 in SUN
+                iter_count = 70 if version >= ChuniConstants.VER_CHUNITHM_SUN else 44
+                hash = PBKDF2(
+                    method_fixed,
+                    bytes.fromhex(keys[2]),
+                    128,
+                    count=iter_count,
+                    hmac_hash_module=SHA1,
+                )
 
-                self.hash_table[version][hash.hex()] = method_fixed
+                hashed_name = hash.hex()[:32] # truncate unused bytes like the game does
+                self.hash_table[version][hashed_name] = method_fixed
 
-                self.logger.debug(f"Hashed v{version} method {method_fixed} with {bytes.fromhex(keys[2])} to get {hash.hex()}")
+                self.logger.debug(
+                    f"Hashed v{version} method {method_fixed} with {bytes.fromhex(keys[2])} to get {hash.hex()}"
+                )
 
     @classmethod
     def get_allnet_info(
@@ -133,30 +150,32 @@ class ChuniServlet:
 
         if version < 105:  # 1.0
             internal_ver = ChuniConstants.VER_CHUNITHM
-        elif version >= 105 and version < 110:  # Plus
+        elif version >= 105 and version < 110:  # PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_PLUS
-        elif version >= 110 and version < 115:  # Air
+        elif version >= 110 and version < 115:  # AIR
             internal_ver = ChuniConstants.VER_CHUNITHM_AIR
-        elif version >= 115 and version < 120:  # Air Plus
+        elif version >= 115 and version < 120:  # AIR PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_AIR_PLUS
-        elif version >= 120 and version < 125:  # Star
+        elif version >= 120 and version < 125:  # STAR
             internal_ver = ChuniConstants.VER_CHUNITHM_STAR
-        elif version >= 125 and version < 130:  # Star Plus
+        elif version >= 125 and version < 130:  # STAR PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_STAR_PLUS
-        elif version >= 130 and version < 135:  # Amazon
+        elif version >= 130 and version < 135:  # AMAZON
             internal_ver = ChuniConstants.VER_CHUNITHM_AMAZON
-        elif version >= 135 and version < 140:  # Amazon Plus
+        elif version >= 135 and version < 140:  # AMAZON PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_AMAZON_PLUS
-        elif version >= 140 and version < 145:  # Crystal
+        elif version >= 140 and version < 145:  # CRYSTAL
             internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL
-        elif version >= 145 and version < 150:  # Crystal Plus
+        elif version >= 145 and version < 150:  # CRYSTAL PLUS
             internal_ver = ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS
-        elif version >= 150 and version < 200:  # Paradise
+        elif version >= 150 and version < 200:  # PARADISE
             internal_ver = ChuniConstants.VER_CHUNITHM_PARADISE
-        elif version >= 200 and version < 205:  # New
+        elif version >= 200 and version < 205:  # NEW!!
             internal_ver = ChuniConstants.VER_CHUNITHM_NEW
-        elif version >= 205 and version < 210:  # New Plus
+        elif version >= 205 and version < 210:  # NEW PLUS!!
             internal_ver = ChuniConstants.VER_CHUNITHM_NEW_PLUS
+        elif version >= 210:  # SUN
+            internal_ver = ChuniConstants.VER_CHUNITHM_SUN
 
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
             # If we get a 32 character long hex string, it's a hash and we're
@@ -167,11 +186,15 @@ class ChuniServlet:
 
             else:
                 if internal_ver not in self.hash_table:
-                    self.logger.error(f"v{version} does not support encryption or no keys entered")
+                    self.logger.error(
+                        f"v{version} does not support encryption or no keys entered"
+                    )
                     return zlib.compress(b'{"stat": "0"}')
 
                 elif endpoint.lower() not in self.hash_table[internal_ver]:
-                    self.logger.error(f"No hash found for v{version} endpoint {endpoint}")
+                    self.logger.error(
+                        f"No hash found for v{version} endpoint {endpoint}"
+                    )
                     return zlib.compress(b'{"stat": "0"}')
 
                 endpoint = self.hash_table[internal_ver][endpoint.lower()]
@@ -193,7 +216,11 @@ class ChuniServlet:
 
             encrtped = True
 
-        if not encrtped and self.game_cfg.crypto.encrypted_only and internal_ver >= ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS:
+        if (
+            not encrtped
+            and self.game_cfg.crypto.encrypted_only
+            and internal_ver >= ChuniConstants.VER_CHUNITHM_CRYSTAL_PLUS
+        ):
             self.logger.error(
                 f"Unencrypted v{version} {endpoint} request, but config is set to encrypted only: {req_raw}"
             )
@@ -210,9 +237,7 @@ class ChuniServlet:
 
         req_data = json.loads(unzip)
 
-        self.logger.info(
+        self.logger.info(f"v{version} {endpoint} request from {client_ip}")
-            f"v{version} {endpoint} request from {client_ip}"
-        )
         self.logger.debug(req_data)
 
         func_to_find = "handle_" + inflection.underscore(endpoint) + "_request"

titles/chuni/new.py

@@ -23,41 +23,44 @@ class ChuniNew(ChuniBase):
         self.version = ChuniConstants.VER_CHUNITHM_NEW
 
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        # use UTC time and convert it to JST time by adding +9
+        # matching therefore starts one hour before and lasts for 8 hours
         match_start = datetime.strftime(
-            datetime.now() - timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=8), self.date_time_format
         )
         match_end = datetime.strftime(
-            datetime.now() + timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=16), self.date_time_format
         )
         reboot_start = datetime.strftime(
-            datetime.now() - timedelta(hours=11), self.date_time_format
+            datetime.utcnow() + timedelta(hours=6), self.date_time_format
         )
         reboot_end = datetime.strftime(
-            datetime.now() - timedelta(hours=10), self.date_time_format
+            datetime.utcnow() + timedelta(hours=7), self.date_time_format
         )
         return {
             "gameSetting": {
-                "isMaintenance": "false",
+                "isMaintenance": False,
                 "requestInterval": 10,
                 "rebootStartTime": reboot_start,
                 "rebootEndTime": reboot_end,
-                "isBackgroundDistribute": "false",
+                "isBackgroundDistribute": False,
                 "maxCountCharacter": 300,
                 "maxCountItem": 300,
                 "maxCountMusic": 300,
                 "matchStartTime": match_start,
                 "matchEndTime": match_end,
-                "matchTimeLimit": 99,
+                "matchTimeLimit": 60,
                 "matchErrorLimit": 9999,
                 "romVersion": self.game_cfg.version.version(self.version)["rom"],
                 "dataVersion": self.game_cfg.version.version(self.version)["data"],
                 "matchingUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
                 "matchingUriX": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
+                # might be really important for online battle to connect the cabs via UDP port 50201
                 "udpHolePunchUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
                 "reflectorUri": f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/200/ChuniServlet/",
             },
-            "isDumpUpload": "false",
+            "isDumpUpload": False,
-            "isAou": "false",
+            "isAou": False,
         }
 
     def handle_remove_token_api_request(self, data: Dict) -> Dict:
@@ -468,3 +471,162 @@ class ChuniNew(ChuniBase):
             self.data.item.put_user_print_state(user_id, id=order_id, hasCompleted=True)
 
         return {"returnCode": "1", "apiName": "CMUpsertUserPrintCancelApi"}
+
+    def handle_ping_request(self, data: Dict) -> Dict:
+        # matchmaking ping request
+        return {"returnCode": "1"}
+
+    def handle_begin_matching_api_request(self, data: Dict) -> Dict:
+        room_id = 1
+        # check if there is a free matching room
+        matching_room = self.data.item.get_oldest_free_matching(self.version)
+
+        if matching_room is None:
+            # grab the latest roomId and add 1 for the new room
+            newest_matching = self.data.item.get_newest_matching(self.version)
+            if newest_matching is not None:
+                room_id = newest_matching["roomId"] + 1
+
+            # fix userName WTF8
+            new_member = data["matchingMemberInfo"]
+            new_member["userName"] = self.read_wtf8(new_member["userName"])
+
+            # create the new room with room_id and the current user id (host)
+            # user id is required for the countdown later on
+            self.data.item.put_matching(
+                self.version, room_id, [new_member], user_id=new_member["userId"]
+            )
+
+            # get the newly created matching room
+            matching_room = self.data.item.get_matching(self.version, room_id)
+        else:
+            # a room already exists, so just add the new member to it
+            matching_member_list = matching_room["matchingMemberInfoList"]
+            # fix userName WTF8
+            new_member = data["matchingMemberInfo"]
+            new_member["userName"] = self.read_wtf8(new_member["userName"])
+            matching_member_list.append(new_member)
+
+            # add the updated room to the database, make sure to set isFull correctly!
+            self.data.item.put_matching(
+                self.version,
+                matching_room["roomId"],
+                matching_member_list,
+                user_id=matching_room["user"],
+                is_full=True if len(matching_member_list) >= 4 else False,
+            )
+
+        matching_wait = {
+            "isFinish": False,
+            "restMSec": matching_room["restMSec"],  # in sec
+            "pollingInterval": 1,  # in sec
+            "matchingMemberInfoList": matching_room["matchingMemberInfoList"],
+        }
+
+        return {"roomId": 1, "matchingWaitState": matching_wait}
+
+    def handle_end_matching_api_request(self, data: Dict) -> Dict:
+        matching_room = self.data.item.get_matching(self.version, data["roomId"])
+        members = matching_room["matchingMemberInfoList"]
+
+        # only set the host user to role 1 every other to 0?
+        role_list = [
+            {"role": 1} if m["userId"] == matching_room["user"] else {"role": 0}
+            for m in members
+        ]
+
+        self.data.item.put_matching(
+            self.version,
+            matching_room["roomId"],
+            members,
+            user_id=matching_room["user"],
+            rest_sec=0,  # make sure to always set 0
+            is_full=True,  # and full, so no one can join
+        )
+
+        return {
+            "matchingResult": 1,  # needs to be 1 for successful matching
+            "matchingMemberInfoList": members,
+            # no idea, maybe to differentiate between CPUs and real players?
+            "matchingMemberRoleList": role_list,
+            # TCP/UDP connection?
+            "reflectorUri": f"{self.core_cfg.title.hostname}",
+        }
+
+    def handle_remove_matching_member_api_request(self, data: Dict) -> Dict:
+        # get all matching rooms, because Chuni only returns the userId
+        # not the actual roomId
+        matching_rooms = self.data.item.get_all_matchings(self.version)
+        if matching_rooms is None:
+            return {"returnCode": "1"}
+
+        for room in matching_rooms:
+            old_members = room["matchingMemberInfoList"]
+            new_members = [m for m in old_members if m["userId"] != data["userId"]]
+
+            # if nothing changed go to the next room
+            if len(old_members) == len(new_members):
+                continue
+
+            # if the last user got removed, delete the matching room
+            if len(new_members) <= 0:
+                self.data.item.delete_matching(self.version, room["roomId"])
+            else:
+                # remove the user from the room
+                self.data.item.put_matching(
+                    self.version,
+                    room["roomId"],
+                    new_members,
+                    user_id=room["user"],
+                    rest_sec=room["restMSec"],
+                )
+
+        return {"returnCode": "1"}
+
+    def handle_get_matching_state_api_request(self, data: Dict) -> Dict:
+        polling_interval = 1
+        # get the current active room
+        matching_room = self.data.item.get_matching(self.version, data["roomId"])
+        members = matching_room["matchingMemberInfoList"]
+        rest_sec = matching_room["restMSec"]
+
+        # grab the current member
+        current_member = data["matchingMemberInfo"]
+
+        # only the host user can decrease the countdown
+        if matching_room["user"] == int(current_member["userId"]):
+            # cap the restMSec to 0
+            if rest_sec > 0:
+                rest_sec -= polling_interval
+            else:
+                rest_sec = 0
+
+        # update the members in order to recieve messages
+        for i, member in enumerate(members):
+            if member["userId"] == current_member["userId"]:
+                # replace the old user data with the current user data,
+                # also parse WTF-8 everytime
+                current_member["userName"] = self.read_wtf8(current_member["userName"])
+                members[i] = current_member
+
+        self.data.item.put_matching(
+            self.version,
+            data["roomId"],
+            members,
+            rest_sec=rest_sec,
+            user_id=matching_room["user"],
+        )
+
+        # only add the other members to the list
+        diff_members = [m for m in members if m["userId"] != current_member["userId"]]
+
+        matching_wait = {
+            # makes no difference? Always use False?
+            "isFinish": True if rest_sec == 0 else False,
+            "restMSec": rest_sec,
+            "pollingInterval": polling_interval,
+            # the current user needs to be the first one?
+            "matchingMemberInfoList": [current_member] + diff_members,
+        }
+
+        return {"matchingWaitState": matching_wait}

titles/chuni/newplus.py

@@ -13,8 +13,12 @@ class ChuniNewPlus(ChuniNew):
 
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         ret = super().handle_get_game_setting_api_request(data)
-        ret["gameSetting"]["romVersion"] = self.game_cfg.version.version(self.version)["rom"]
+        ret["gameSetting"]["romVersion"] = self.game_cfg.version.version(self.version)[
-        ret["gameSetting"]["dataVersion"] = self.game_cfg.version.version(self.version)["data"]
+            "rom"
+        ]
+        ret["gameSetting"]["dataVersion"] = self.game_cfg.version.version(self.version)[
+            "data"
+        ]
         ret["gameSetting"][
             "matchingUri"
         ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/205/ChuniServlet/"
@@ -32,6 +36,6 @@ class ChuniNewPlus(ChuniNew):
     def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
         user_data = super().handle_cm_get_user_preview_api_request(data)
 
-        # hardcode lastDataVersion for CardMaker 1.35
+        # hardcode lastDataVersion for CardMaker 1.35 A028
         user_data["lastDataVersion"] = "2.05.00"
         return user_data

titles/chuni/schema/item.py

@@ -1,5 +1,12 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
+from sqlalchemy import (
+    Table,
+    Column,
+    UniqueConstraint,
+    PrimaryKeyConstraint,
+    and_,
+    delete,
+)
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.schema import ForeignKey
@@ -203,8 +210,141 @@ login_bonus = Table(
     mysql_charset="utf8mb4",
 )
 
+favorite = Table(
+    "chuni_item_favorite",
+    metadata,
+    Column("id", Integer, primary_key=True, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("favId", Integer, nullable=False),
+    Column("favKind", Integer, nullable=False, server_default="1"),
+    UniqueConstraint("version", "user", "favId", name="chuni_item_favorite_uk"),
+    mysql_charset="utf8mb4",
+)
+
+matching = Table(
+    "chuni_item_matching",
+    metadata,
+    Column("roomId", Integer, nullable=False),
+    Column(
+        "user",
+        ForeignKey("aime_user.id", ondelete="cascade", onupdate="cascade"),
+        nullable=False,
+    ),
+    Column("version", Integer, nullable=False),
+    Column("restMSec", Integer, nullable=False, server_default="60"),
+    Column("isFull", Boolean, nullable=False, server_default="0"),
+    PrimaryKeyConstraint("roomId", "version", name="chuni_item_matching_pk"),
+    Column("matchingMemberInfoList", JSON, nullable=False),
+    mysql_charset="utf8mb4",
+)
+
 
 class ChuniItemData(BaseData):
+    def get_oldest_free_matching(self, version: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version,
+                matching.c.isFull == False
+            )
+        ).order_by(matching.c.roomId.asc())
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_newest_matching(self, version: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version
+            )
+        ).order_by(matching.c.roomId.desc())
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def get_all_matchings(self, version: int) -> Optional[List[Row]]:
+        sql = matching.select(
+            and_(
+                matching.c.version == version
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
+    def get_matching(self, version: int, room_id: int) -> Optional[Row]:
+        sql = matching.select(
+            and_(matching.c.version == version, matching.c.roomId == room_id)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchone()
+
+    def put_matching(
+        self,
+        version: int,
+        room_id: int,
+        matching_member_info_list: list,
+        user_id: int = None,
+        rest_sec: int = 60,
+        is_full: bool = False
+    ) -> Optional[int]:
+        sql = insert(matching).values(
+            roomId=room_id,
+            version=version,
+            restMSec=rest_sec,
+            user=user_id,
+            isFull=is_full,
+            matchingMemberInfoList=matching_member_info_list,
+        )
+
+        conflict = sql.on_duplicate_key_update(
+            restMSec=rest_sec, matchingMemberInfoList=matching_member_info_list
+        )
+
+        result = self.execute(conflict)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def delete_matching(self, version: int, room_id: int):
+        sql = delete(matching).where(
+            and_(matching.c.roomId == room_id, matching.c.version == version)
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.lastrowid
+
+    def get_all_favorites(
+        self, user_id: int, version: int, fav_kind: int = 1
+    ) -> Optional[List[Row]]:
+        sql = favorite.select(
+            and_(
+                favorite.c.version == version,
+                favorite.c.user == user_id,
+                favorite.c.favKind == fav_kind,
+            )
+        )
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def put_login_bonus(
         self, user_id: int, version: int, preset_id: int, **login_bonus_data
     ) -> Optional[int]:

titles/chuni/schema/profile.py

@@ -89,8 +89,6 @@ profile = Table(
         Integer,
         ForeignKey("chuni_profile_team.id", ondelete="SET NULL", onupdate="SET NULL"),
     ),
-    Column("avatarBack", Integer, server_default="0"),
-    Column("avatarFace", Integer, server_default="0"),
     Column("eliteRankPoint", Integer, server_default="0"),
     Column("stockedGridCount", Integer, server_default="0"),
     Column("netBattleLoseCount", Integer, server_default="0"),
@@ -98,10 +96,8 @@ profile = Table(
     Column("netBattle4thCount", Integer, server_default="0"),
     Column("overPowerRate", Integer, server_default="0"),
     Column("battleRewardStatus", Integer, server_default="0"),
-    Column("avatarPoint", Integer, server_default="0"),
     Column("netBattle1stCount", Integer, server_default="0"),
     Column("charaIllustId", Integer, server_default="0"),
-    Column("avatarItem", Integer, server_default="0"),
     Column("userNameEx", String(8), server_default=""),
     Column("netBattleWinCount", Integer, server_default="0"),
     Column("netBattleCorrection", Integer, server_default="0"),
@@ -112,7 +108,6 @@ profile = Table(
     Column("netBattle3rdCount", Integer, server_default="0"),
     Column("netBattleConsecutiveWinCount", Integer, server_default="0"),
     Column("overPowerLowerRank", Integer, server_default="0"),
-    Column("avatarWear", Integer, server_default="0"),
     Column("classEmblemBase", Integer, server_default="0"),
     Column("battleRankPoint", Integer, server_default="0"),
     Column("netBattle2ndCount", Integer, server_default="0"),
@@ -120,13 +115,19 @@ profile = Table(
     Column("skillId", Integer, server_default="0"),
     Column("lastCountryCode", String(5), server_default="JPN"),
     Column("isNetBattleHost", Boolean, server_default="0"),
-    Column("avatarFront", Integer, server_default="0"),
-    Column("avatarSkin", Integer, server_default="0"),
     Column("battleRewardCount", Integer, server_default="0"),
     Column("battleRewardIndex", Integer, server_default="0"),
     Column("netBattlePlayCount", Integer, server_default="0"),
     Column("exMapLoopCount", Integer, server_default="0"),
     Column("netBattleEndState", Integer, server_default="0"),
+    Column("rankUpChallengeResults", JSON),
+    Column("avatarBack", Integer, server_default="0"),
+    Column("avatarFace", Integer, server_default="0"),
+    Column("avatarPoint", Integer, server_default="0"),
+    Column("avatarItem", Integer, server_default="0"),
+    Column("avatarWear", Integer, server_default="0"),
+    Column("avatarFront", Integer, server_default="0"),
+    Column("avatarSkin", Integer, server_default="0"),
     Column("avatarHead", Integer, server_default="0"),
     UniqueConstraint("user", "version", name="chuni_profile_profile_uk"),
     mysql_charset="utf8mb4",
@@ -417,8 +418,8 @@ class ChuniProfileData(BaseData):
         sql = (
             select([profile, option])
             .join(option, profile.c.user == option.c.user)
-            .filter(and_(profile.c.user == aime_id, profile.c.version == version))
+            .filter(and_(profile.c.user == aime_id, profile.c.version <= version))
-        )
+        ).order_by(profile.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -429,9 +430,9 @@ class ChuniProfileData(BaseData):
         sql = select(profile).where(
             and_(
                 profile.c.user == aime_id,
-                profile.c.version == version,
+                profile.c.version <= version,
             )
-        )
+        ).order_by(profile.c.version.desc())
 
         result = self.execute(sql)
         if result is None:
@@ -461,9 +462,9 @@ class ChuniProfileData(BaseData):
         sql = select(profile_ex).where(
             and_(
                 profile_ex.c.user == aime_id,
-                profile_ex.c.version == version,
+                profile_ex.c.version <= version,
             )
-        )
+        ).order_by(profile_ex.c.version.desc())
 
         result = self.execute(sql)
         if result is None:

titles/chuni/schema/score.py

@@ -134,7 +134,9 @@ playlog = Table(
     Column("charaIllustId", Integer),
     Column("romVersion", String(255)),
     Column("judgeHeaven", Integer),
-    mysql_charset="utf8mb4",
+    Column("regionId", Integer),
+    Column("machineType", Integer),
+    mysql_charset="utf8mb4"
 )
 
 

titles/chuni/schema/static.py

@@ -1,11 +1,19 @@
 from typing import Dict, List, Optional
-from sqlalchemy import Table, Column, UniqueConstraint, PrimaryKeyConstraint, and_
+from sqlalchemy import (
+    ForeignKeyConstraint,
+    Table,
+    Column,
+    UniqueConstraint,
+    PrimaryKeyConstraint,
+    and_,
+)
 from sqlalchemy.types import Integer, String, TIMESTAMP, Boolean, JSON, Float
 from sqlalchemy.engine.base import Connection
 from sqlalchemy.engine import Row
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.sql import func, select
 from sqlalchemy.dialects.mysql import insert
+from datetime import datetime
 
 from core.data.schema import BaseData, metadata
 
@@ -17,6 +25,7 @@ events = Table(
     Column("eventId", Integer),
     Column("type", Integer),
     Column("name", String(255)),
+    Column("startDate", TIMESTAMP, server_default=func.now()),
     Column("enabled", Boolean, server_default="1"),
     UniqueConstraint("version", "eventId", name="chuni_static_events_uk"),
     mysql_charset="utf8mb4",
@@ -125,11 +134,13 @@ gacha_cards = Table(
 login_bonus_preset = Table(
     "chuni_static_login_bonus_preset",
     metadata,
-    Column("id", Integer, primary_key=True, nullable=False),
+    Column("presetId", Integer, nullable=False),
     Column("version", Integer, nullable=False),
     Column("presetName", String(255), nullable=False),
     Column("isEnabled", Boolean, server_default="1"),
-    UniqueConstraint("version", "id", name="chuni_static_login_bonus_preset_uk"),
+    PrimaryKeyConstraint(
+        "presetId", "version", name="chuni_static_login_bonus_preset_pk"
+    ),
     mysql_charset="utf8mb4",
 )
 
@@ -138,15 +149,7 @@ login_bonus = Table(
     metadata,
     Column("id", Integer, primary_key=True, nullable=False),
     Column("version", Integer, nullable=False),
-    Column(
+    Column("presetId", Integer, nullable=False),
-        "presetId",
-        ForeignKey(
-            "chuni_static_login_bonus_preset.id",
-            ondelete="cascade",
-            onupdate="cascade",
-        ),
-        nullable=False,
-    ),
     Column("loginBonusId", Integer, nullable=False),
     Column("loginBonusName", String(255), nullable=False),
     Column("presentId", Integer, nullable=False),
@@ -157,6 +160,16 @@ login_bonus = Table(
     UniqueConstraint(
         "version", "presetId", "loginBonusId", name="chuni_static_login_bonus_uk"
     ),
+    ForeignKeyConstraint(
+        ["presetId", "version"],
+        [
+            "chuni_static_login_bonus_preset.presetId",
+            "chuni_static_login_bonus_preset.version",
+        ],
+        onupdate="CASCADE",
+        ondelete="CASCADE",
+        name="chuni_static_login_bonus_ibfk_1",
+    ),
     mysql_charset="utf8mb4",
 )
 
@@ -200,7 +213,9 @@ class ChuniStaticData(BaseData):
         return result.lastrowid
 
     def get_login_bonus(
-        self, version: int, preset_id: int,
+        self,
+        version: int,
+        preset_id: int,
     ) -> Optional[List[Row]]:
         sql = login_bonus.select(
             and_(
@@ -234,7 +249,7 @@ class ChuniStaticData(BaseData):
         self, version: int, preset_id: int, preset_name: str, is_enabled: bool
     ) -> Optional[int]:
         sql = insert(login_bonus_preset).values(
-            id=preset_id,
+            presetId=preset_id,
             version=version,
             presetName=preset_name,
             isEnabled=is_enabled,
@@ -414,6 +429,14 @@ class ChuniStaticData(BaseData):
             return None
         return result.fetchall()
 
+    def get_music(self, version: int) -> Optional[List[Row]]:
+        sql = music.select(music.c.version <= version)
+
+        result = self.execute(sql)
+        if result is None:
+            return None
+        return result.fetchall()
+
     def get_music_chart(
         self, version: int, song_id: int, chart_id: int
     ) -> Optional[List[Row]]:

titles/chuni/sun.py

@@ -0,0 +1,37 @@
+from typing import Dict, Any
+
+from core.config import CoreConfig
+from titles.chuni.newplus import ChuniNewPlus
+from titles.chuni.const import ChuniConstants
+from titles.chuni.config import ChuniConfig
+
+
+class ChuniSun(ChuniNewPlus):
+    def __init__(self, core_cfg: CoreConfig, game_cfg: ChuniConfig) -> None:
+        super().__init__(core_cfg, game_cfg)
+        self.version = ChuniConstants.VER_CHUNITHM_SUN
+
+    def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
+        ret = super().handle_get_game_setting_api_request(data)
+        ret["gameSetting"]["romVersion"] = self.game_cfg.version.version(self.version)["rom"]
+        ret["gameSetting"]["dataVersion"] = self.game_cfg.version.version(self.version)["data"]
+        ret["gameSetting"][
+            "matchingUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "matchingUriX"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "udpHolePunchUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        ret["gameSetting"][
+            "reflectorUri"
+        ] = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}/SDHD/210/ChuniServlet/"
+        return ret
+
+    def handle_cm_get_user_preview_api_request(self, data: Dict) -> Dict:
+        user_data = super().handle_cm_get_user_preview_api_request(data)
+
+        # hardcode lastDataVersion for CardMaker 1.35 A032
+        user_data["lastDataVersion"] = "2.10.00"
+        return user_data

titles/cm/base.py

@@ -23,19 +23,40 @@ class CardMakerBase:
         self.game = CardMakerConstants.GAME_CODE
         self.version = CardMakerConstants.VER_CARD_MAKER
 
+    @staticmethod
+    def _parse_int_ver(version: str) -> str:
+        return version.replace(".", "")[:3]
+
     def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
         if self.core_cfg.server.is_develop:
             uri = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}"
         else:
             uri = f"http://{self.core_cfg.title.hostname}"
 
-        # CHUNITHM = 0, maimai = 1, ONGEKI = 2
+        # grab the dict with all games version numbers from user config
+        games_ver = self.game_cfg.version.version(self.version)
+
         return {
             "length": 3,
             "gameConnectList": [
-                {"modelKind": 0, "type": 1, "titleUri": f"{uri}/SDHD/200/"},
+                # CHUNITHM
-                {"modelKind": 1, "type": 1, "titleUri": f"{uri}/SDEZ/120/"},
+                {
-                {"modelKind": 2, "type": 1, "titleUri": f"{uri}/SDDT/130/"},
+                    "modelKind": 0,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDHD/{self._parse_int_ver(games_ver['chuni'])}/",
+                },
+                # maimai DX
+                {
+                    "modelKind": 1,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDEZ/{self._parse_int_ver(games_ver['maimai'])}/",
+                },
+                # ONGEKI
+                {
+                    "modelKind": 2,
+                    "type": 1,
+                    "titleUri": f"{uri}/SDDT/{self._parse_int_ver(games_ver['ongeki'])}/",
+                },
             ],
         }
 
@@ -47,12 +68,15 @@ class CardMakerBase:
             datetime.now() + timedelta(hours=4), self.date_time_format
         )
 
+        # grab the dict with all games version numbers from user config
+        games_ver = self.game_cfg.version.version(self.version)
+
         return {
             "gameSetting": {
                 "dataVersion": "1.30.00",
-                "ongekiCmVersion": "1.30.01",
+                "ongekiCmVersion": games_ver["ongeki"],
-                "chuniCmVersion": "2.00.00",
+                "chuniCmVersion": games_ver["chuni"],
-                "maimaiCmVersion": "1.20.00",
+                "maimaiCmVersion": games_ver["maimai"],
                 "requestInterval": 10,
                 "rebootStartTime": reboot_start,
                 "rebootEndTime": reboot_end,

titles/cm/cm135.py

@@ -1,8 +1,4 @@
-from datetime import date, datetime, timedelta
+from typing import Dict
-from typing import Any, Dict, List
-import json
-import logging
-from enum import Enum
 
 from core.config import CoreConfig
 from core.data.cache import cached
@@ -16,23 +12,7 @@ class CardMaker135(CardMakerBase):
         super().__init__(core_cfg, game_cfg)
         self.version = CardMakerConstants.VER_CARD_MAKER_135
 
-    def handle_get_game_connect_api_request(self, data: Dict) -> Dict:
-        ret = super().handle_get_game_connect_api_request(data)
-        if self.core_cfg.server.is_develop:
-            uri = f"http://{self.core_cfg.title.hostname}:{self.core_cfg.title.port}"
-        else:
-            uri = f"http://{self.core_cfg.title.hostname}"
-
-        ret["gameConnectList"][0]["titleUri"] = f"{uri}/SDHD/205/"
-        ret["gameConnectList"][1]["titleUri"] = f"{uri}/SDEZ/125/"
-        ret["gameConnectList"][2]["titleUri"] = f"{uri}/SDDT/135/"
-
-        return ret
-
     def handle_get_game_setting_api_request(self, data: Dict) -> Dict:
         ret = super().handle_get_game_setting_api_request(data)
         ret["gameSetting"]["dataVersion"] = "1.35.00"
-        ret["gameSetting"]["ongekiCmVersion"] = "1.35.03"
-        ret["gameSetting"]["chuniCmVersion"] = "2.05.00"
-        ret["gameSetting"]["maimaiCmVersion"] = "1.25.00"
         return ret

titles/cm/config.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from core.config import CoreConfig
 
 
@@ -20,6 +21,21 @@ class CardMakerServerConfig:
         )
 
 
+class CardMakerVersionConfig:
+    def __init__(self, parent_config: "CardMakerConfig") -> None:
+        self.__config = parent_config
+
+    def version(self, version: int) -> Dict:
+        """
+        in the form of:
+        1: {"ongeki": 1.30.01, "chuni": 2.00.00, "maimai": 1.20.00}
+        """
+        return CoreConfig.get_config_field(
+            self.__config, "cardmaker", "version", default={}
+        )[version]
+
+
 class CardMakerConfig(dict):
     def __init__(self) -> None:
         self.server = CardMakerServerConfig(self)
+        self.version = CardMakerVersionConfig(self)

titles/cm/const.py

@@ -6,7 +6,7 @@ class CardMakerConstants:
     VER_CARD_MAKER = 0
     VER_CARD_MAKER_135 = 1
 
-    VERSION_NAMES = ("Card Maker 1.34", "Card Maker 1.35")
+    VERSION_NAMES = ("Card Maker 1.30", "Card Maker 1.35")
 
     @classmethod
     def game_ver_to_string(cls, ver: int):

titles/cm/index.py

@@ -30,7 +30,7 @@ class CardMakerServlet:
 
         self.versions = [
             CardMakerBase(core_cfg, self.game_cfg),
-            CardMaker135(core_cfg, self.game_cfg),
+            CardMaker135(core_cfg, self.game_cfg)
         ]
 
         self.logger = logging.getLogger("cardmaker")
@@ -89,7 +89,7 @@ class CardMakerServlet:
 
         if version >= 130 and version < 135:  # Card Maker
             internal_ver = CardMakerConstants.VER_CARD_MAKER
-        elif version >= 135 and version < 136:  # Card Maker 1.35
+        elif version >= 135 and version < 140:  # Card Maker 1.35
             internal_ver = CardMakerConstants.VER_CARD_MAKER_135
 
         if all(c in string.hexdigits for c in endpoint) and len(endpoint) == 32:
@@ -109,9 +109,7 @@ class CardMakerServlet:
 
         req_data = json.loads(unzip)
 
-        self.logger.info(
+        self.logger.info(f"v{version} {endpoint} request from {client_ip}")
-            f"v{version} {endpoint} request from {client_ip}"
-        )
         self.logger.debug(req_data)
 
         func_to_find = "handle_" + inflection.underscore(endpoint) + "_request"

titles/cm/read.py

@@ -90,7 +90,7 @@ class CardMakerReader(BaseReader):
             "v2_00": ChuniConstants.VER_CHUNITHM_NEW,
             "v2_05": ChuniConstants.VER_CHUNITHM_NEW_PLUS,
             # Chunithm SUN, ignore for now
-            "v2_10": ChuniConstants.VER_CHUNITHM_NEW_PLUS + 1
+            "v2_10": ChuniConstants.VER_CHUNITHM_NEW_PLUS + 1,
         }
 
         for root, dirs, files in os.walk(base_dir):

titles/cxb/index.py

@@ -101,9 +101,7 @@ class CxbServlet(resource.Resource):
                     f"Ready on ports {self.game_cfg.server.port} & {self.game_cfg.server.port_secure}"
                 )
             else:
-                self.logger.info(
+                self.logger.info(f"Ready on port {self.game_cfg.server.port}")
-                    f"Ready on port {self.game_cfg.server.port}"
-                )
 
     def render_POST(self, request: Request):
         version = 0

titles/diva/__init__.py

@@ -7,4 +7,4 @@ index = DivaServlet
 database = DivaData
 reader = DivaReader
 game_codes = [DivaConstants.GAME_CODE]
-current_schema_version = 1
+current_schema_version = 5

titles/diva/base.py

@@ -266,16 +266,17 @@ class DivaBase:
     def handle_festa_info_request(self, data: Dict) -> Dict:
         encoded = "&"
         params = {
-            "fi_id": "1,-1",
+            "fi_id": "1,2",
-            "fi_name": f"{self.core_cfg.server.name} Opening,xxx",
+            "fi_name": f"{self.core_cfg.server.name} Opening,Project DIVA Festa",
-            "fi_kind": "0,0",
+            # 0=PINK, 1=GREEN
+            "fi_kind": "1,0",
             "fi_difficulty": "-1,-1",
             "fi_pv_id_lst": "ALL,ALL",
             "fi_attr": "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF,7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
-            "fi_add_vp": "20,0",
+            "fi_add_vp": "20,5",
-            "fi_mul_vp": "1,1",
+            "fi_mul_vp": "1,2",
-            "fi_st": "2022-06-17 17:00:00.0,2014-07-08 18:10:11.0",
+            "fi_st": "2019-01-01 00:00:00.0,2019-01-01 00:00:00.0",
-            "fi_et": "2029-01-01 10:00:00.0,2014-07-08 18:10:11.0",
+            "fi_et": "2029-01-01 00:00:00.0,2029-01-01 00:00:00.0",
             "fi_lut": "{self.time_lut}",
         }
 
@@ -401,10 +402,10 @@ class DivaBase:
             response += f"&lv_pnt={profile['lv_pnt']}"
             response += f"&vcld_pts={profile['vcld_pts']}"
             response += f"&skn_eqp={profile['use_pv_skn_eqp']}"
-            response += f"&btn_se_eqp={profile['use_pv_btn_se_eqp']}"
+            response += f"&btn_se_eqp={profile['btn_se_eqp']}"
-            response += f"&sld_se_eqp={profile['use_pv_sld_se_eqp']}"
+            response += f"&sld_se_eqp={profile['sld_se_eqp']}"
-            response += f"&chn_sld_se_eqp={profile['use_pv_chn_sld_se_eqp']}"
+            response += f"&chn_sld_se_eqp={profile['chn_sld_se_eqp']}"
-            response += f"&sldr_tch_se_eqp={profile['use_pv_sldr_tch_se_eqp']}"
+            response += f"&sldr_tch_se_eqp={profile['sldr_tch_se_eqp']}"
             response += f"&passwd_stat={profile['passwd_stat']}"
 
             # Store stuff to add to rework
@@ -478,6 +479,21 @@ class DivaBase:
         response += f"&dsp_clr_sts={profile['dsp_clr_sts']}"
         response += f"&rgo_sts={profile['rgo_sts']}"
 
+        # Contest progress
+        response += f"&cv_cid=-1,-1,-1,-1"
+        response += f"&cv_sc=-1,-1,-1,-1"
+        response += f"&cv_bv=-1,-1,-1,-1"
+        response += f"&cv_bv=-1,-1,-1,-1"
+        response += f"&cv_bf=-1,-1,-1,-1"
+
+        # Contest now playing id, return -1 if no current playing contest
+        response += f"&cnp_cid={profile['cnp_cid']}"
+        response += f"&cnp_val={profile['cnp_val']}"
+        # border can be 0=bronzem 1=silver, 2=gold
+        response += f"&cnp_rr={profile['cnp_rr']}"
+        # only show contest specifier if it is not empty
+        response += f"&cnp_sp={profile['cnp_sp']}" if profile["cnp_sp"] != "" else ""
+
         # To be fully fixed
         if "my_qst_id" not in profile:
             response += f"&my_qst_id=-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1"
@@ -488,7 +504,63 @@ class DivaBase:
 
         response += f"&my_qst_prgrs=0,0,0,0,0,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1"
         response += f"&my_qst_et=2022-06-19%2010%3A28%3A52.0,2022-06-19%2010%3A28%3A52.0,2022-06-19%2010%3A28%3A52.0,2100-01-01%2008%3A59%3A59.0,2100-01-01%2008%3A59%3A59.0,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx,xxx"
-        response += f"&clr_sts=0,0,0,0,0,0,0,0,56,52,35,6,6,3,1,0,0,0,0,0"
+
+        # define a helper class to store all counts for clear, great,
+        # excellent and perfect
+        class ClearSet:
+            def __init__(self):
+                self.clear = 0
+                self.great = 0
+                self.excellent = 0
+                self.perfect = 0
+
+        # create a dict to store the ClearSets per difficulty
+        clear_set_dict = {
+            0: ClearSet(),  # easy
+            1: ClearSet(),  # normal
+            2: ClearSet(),  # hard
+            3: ClearSet(),  # extreme
+            4: ClearSet(),  # exExtreme
+        }
+
+        # get clear status from user scores
+        pv_records = self.data.score.get_best_scores(data["pd_id"])
+        clear_status = "0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0"
+
+        if pv_records is not None:
+            for score in pv_records:
+                if score["edition"] == 0:
+                    # cheap and standard both count to "clear"
+                    if score["clr_kind"] in {1, 2}:
+                        clear_set_dict[score["difficulty"]].clear += 1
+                    elif score["clr_kind"] == 3:
+                        clear_set_dict[score["difficulty"]].great += 1
+                    elif score["clr_kind"] == 4:
+                        clear_set_dict[score["difficulty"]].excellent += 1
+                    elif score["clr_kind"] == 5:
+                        clear_set_dict[score["difficulty"]].perfect += 1
+                else:
+                    # 4=ExExtreme
+                    if score["clr_kind"] in {1, 2}:
+                        clear_set_dict[4].clear += 1
+                    elif score["clr_kind"] == 3:
+                        clear_set_dict[4].great += 1
+                    elif score["clr_kind"] == 4:
+                        clear_set_dict[4].excellent += 1
+                    elif score["clr_kind"] == 5:
+                        clear_set_dict[4].perfect += 1
+
+            # now add all values to a list
+            clear_list = []
+            for clear_set in clear_set_dict.values():
+                clear_list.append(clear_set.clear)
+                clear_list.append(clear_set.great)
+                clear_list.append(clear_set.excellent)
+                clear_list.append(clear_set.perfect)
+
+            clear_status = ",".join(map(str, clear_list))
+
+        response += f"&clr_sts={clear_status}"
 
         # Store stuff to add to rework
         response += f"&mdl_eqp_tm={self.time_lut}"

titles/diva/schema/profile.py

@@ -34,9 +34,17 @@ profile = Table(
     Column("use_pv_sld_se_eqp", Boolean, nullable=False, server_default="0"),
     Column("use_pv_chn_sld_se_eqp", Boolean, nullable=False, server_default="0"),
     Column("use_pv_sldr_tch_se_eqp", Boolean, nullable=False, server_default="0"),
+    Column("btn_se_eqp", Integer, nullable=False, server_default="-1"),
+    Column("sld_se_eqp", Integer, nullable=False, server_default="-1"),
+    Column("chn_sld_se_eqp", Integer, nullable=False, server_default="-1"),
+    Column("sldr_tch_se_eqp", Integer, nullable=False, server_default="-1"),
     Column("nxt_pv_id", Integer, nullable=False, server_default="708"),
     Column("nxt_dffclty", Integer, nullable=False, server_default="2"),
     Column("nxt_edtn", Integer, nullable=False, server_default="0"),
+    Column("cnp_cid", Integer, nullable=False, server_default="-1"),
+    Column("cnp_val", Integer, nullable=False, server_default="-1"),
+    Column("cnp_rr", Integer, nullable=False, server_default="-1"),
+    Column("cnp_sp", String(255), nullable=False, server_default=""),
     Column("dsp_clr_brdr", Integer, nullable=False, server_default="7"),
     Column("dsp_intrm_rnk", Integer, nullable=False, server_default="1"),
     Column("dsp_clr_sts", Integer, nullable=False, server_default="1"),

titles/diva/schema/score.py

@@ -3,6 +3,7 @@ from sqlalchemy.types import Integer, String, TIMESTAMP, JSON, Boolean
 from sqlalchemy.schema import ForeignKey
 from sqlalchemy.sql import func, select
 from sqlalchemy.dialects.mysql import insert
+from sqlalchemy.engine import Row
 from typing import Optional, List, Dict, Any
 
 from core.data.schema import BaseData, metadata
@@ -167,7 +168,7 @@ class DivaScoreData(BaseData):
 
     def get_best_user_score(
         self, user_id: int, pv_id: int, difficulty: int, edition: int
-    ) -> Optional[Dict]:
+    ) -> Optional[Row]:
         sql = score.select(
             and_(
                 score.c.user == user_id,
@@ -184,7 +185,7 @@ class DivaScoreData(BaseData):
 
     def get_top3_scores(
         self, pv_id: int, difficulty: int, edition: int
-    ) -> Optional[List[Dict]]:
+    ) -> Optional[List[Row]]:
         sql = (
             score.select(
                 and_(
@@ -204,7 +205,7 @@ class DivaScoreData(BaseData):
 
     def get_global_ranking(
         self, user_id: int, pv_id: int, difficulty: int, edition: int
-    ) -> Optional[List]:
+    ) -> Optional[List[Row]]:
         # get the subquery max score of a user with pv_id, difficulty and
         # edition
         sql_sub = (
@@ -231,7 +232,7 @@ class DivaScoreData(BaseData):
             return None
         return result.fetchone()
 
-    def get_best_scores(self, user_id: int) -> Optional[List]:
+    def get_best_scores(self, user_id: int) -> Optional[List[Row]]:
         sql = score.select(score.c.user == user_id)
 
         result = self.execute(sql)

titles/idz/__init__.py

@@ -0,0 +1,8 @@
+from titles.idz.index import IDZServlet
+from titles.idz.const import IDZConstants
+from titles.idz.database import IDZData
+
+index = IDZServlet
+database = IDZData
+game_codes = [IDZConstants.GAME_CODE]
+current_schema_version = 1

titles/idz/config.py

@@ -0,0 +1,73 @@
+from typing import List, Dict
+
+from core.config import CoreConfig
+
+
+class IDZServerConfig:
+    def __init__(self, parent_config: "IDZConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def enable(self) -> bool:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "server", "enable", default=True
+        )
+
+    @property
+    def loglevel(self) -> int:
+        return CoreConfig.str_to_loglevel(
+            CoreConfig.get_config_field(
+                self.__config, "idz", "server", "loglevel", default="info"
+            )
+        )
+
+    @property
+    def hostname(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "server", "hostname", default=""
+        )
+
+    @property
+    def news(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "server", "news", default=""
+        )
+
+    @property
+    def aes_key(self) -> str:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "server", "aes_key", default=""
+        )
+
+
+class IDZPortsConfig:
+    def __init__(self, parent_config: "IDZConfig") -> None:
+        self.__config = parent_config
+
+    @property
+    def userdb(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "ports", "userdb", default=10000
+        )
+
+    @property
+    def match(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "ports", "match", default=10010
+        )
+
+    @property
+    def echo(self) -> int:
+        return CoreConfig.get_config_field(
+            self.__config, "idz", "ports", "echo", default=10020
+        )
+
+
+class IDZConfig(dict):
+    def __init__(self) -> None:
+        self.server = IDZServerConfig(self)
+        self.ports = IDZPortsConfig(self)
+
+    @property
+    def rsa_keys(self) -> List[Dict]:
+        return CoreConfig.get_config_field(self, "idz", "rsa_keys", default=[])

titles/idz/const.py

@@ -0,0 +1,53 @@
+from enum import Enum
+
+
+class IDZConstants:
+    GAME_CODE = "SDDF"
+
+    CONFIG_NAME = "idz.yaml"
+
+    VER_IDZ_110 = 0
+    VER_IDZ_130 = 1
+    VER_IDZ_210 = 2
+    VER_IDZ_230 = 3
+    NUM_VERS = 4
+
+    VERSION_NAMES = (
+        "Initial D Arcade Stage Zero v1.10",
+        "Initial D Arcade Stage Zero v1.30",
+        "Initial D Arcade Stage Zero v2.10",
+        "Initial D Arcade Stage Zero v2.30",
+    )
+
+    class PROFILE_STATUS(Enum):
+        LOCKED = 0
+        UNLOCKED = 1
+        OLD = 2
+
+    HASH_LUT = [
+        # No clue
+        0x9C82E674,
+        0x5A4738D9,
+        0x8B8D7AE0,
+        0x29EC9D81,
+        # These three are from AES TE0
+        0x1209091B,
+        0x1D83839E,
+        0x582C2C74,
+        0x341A1A2E,
+        0x361B1B2D,
+        0xDC6E6EB2,
+        0xB45A5AEE,
+        0x5BA0A0FB,
+        0xA45252F6,
+        0x763B3B4D,
+        0xB7D6D661,
+        0x7DB3B3CE,
+    ]
+    HASH_NUM = 0
+    HASH_MUL = [5, 7, 11, 12][HASH_NUM]
+    HASH_XOR = [0xB3, 0x8C, 0x14, 0x50][HASH_NUM]
+
+    @classmethod
+    def game_ver_to_string(cls, ver: int):
+        return cls.VERSION_NAMES[ver]

titles/idz/database.py

@@ -0,0 +1,7 @@
+from core.data import Data
+from core.config import CoreConfig
+
+
+class IDZData(Data):
+    def __init__(self, cfg: CoreConfig) -> None:
+        super().__init__(cfg)

titles/idz/echo.py

@@ -0,0 +1,19 @@
+from twisted.internet.protocol import DatagramProtocol
+import logging
+
+from core.config import CoreConfig
+from .config import IDZConfig
+
+
+class IDZEcho(DatagramProtocol):
+    def __init__(self, cfg: CoreConfig, game_cfg: IDZConfig) -> None:
+        super().__init__()
+        self.core_config = cfg
+        self.game_config = game_cfg
+        self.logger = logging.getLogger("idz")
+
+    def datagramReceived(self, data, addr):
+        self.logger.debug(
+            f"Echo from from {addr[0]}:{addr[1]} -> {self.transport.getHost().port} - {data.hex()}"
+        )
+        self.transport.write(data, addr)

titles/idz/handlers/__init__.py

@@ -0,0 +1,47 @@
+from .base import IDZHandlerBase
+
+from .load_server_info import IDZHandlerLoadServerInfo
+
+from .load_ghost import IDZHandlerLoadGhost
+
+from .load_config import IDZHandlerLoadConfigA, IDZHandlerLoadConfigB
+
+from .load_top_ten import IDZHandlerLoadTopTen
+
+from .update_story_clear_num import IDZHandlerUpdateStoryClearNum
+
+from .save_expedition import IDZHandlerSaveExpedition
+
+from .load_2on2 import IDZHandlerLoad2on2A, IDZHandlerLoad2on2B
+
+from .load_team_ranking import IDZHandlerLoadTeamRankingA, IDZHandlerLoadTeamRankingB
+
+from .discover_profile import IDZHandlerDiscoverProfile
+
+from .lock_profile import IDZHandlerLockProfile
+
+from .check_team_names import IDZHandlerCheckTeamName
+
+from .unknown import IDZHandlerUnknown
+
+from .create_profile import IDZHandlerCreateProfile
+
+from .create_auto_team import IDZHandlerCreateAutoTeam
+
+from .load_profile import IDZHandlerLoadProfile
+
+from .save_profile import IDZHandlerSaveProfile
+
+from .update_provisional_store_rank import IDZHandlerUpdateProvisionalStoreRank
+
+from .load_reward_table import IDZHandlerLoadRewardTable
+
+from .save_topic import IDZHandlerSaveTopic
+
+from .save_time_attack import IDZHandlerSaveTimeAttack
+
+from .unlock_profile import IDZHandlerUnlockProfile
+
+from .update_team_points import IDZHandleUpdateTeamPoints
+
+from .update_ui_report import IDZHandleUpdateUIReport

titles/idz/handlers/base.py

@@ -0,0 +1,26 @@
+import logging
+import struct
+from core.data import Data
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerBase:
+    name = "generic"
+    cmd_codes = [0x0000] * IDZConstants.NUM_VERS
+    rsp_codes = [0x0001] * IDZConstants.NUM_VERS
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        self.core_config = core_cfg
+        self.game_cfg = game_cfg
+        self.data = Data(core_cfg)
+        self.logger = logging.getLogger("idz")
+        self.game = IDZConstants.GAME_CODE
+        self.version = version
+        self.size = 0x30
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = bytearray([0] * self.size)
+        struct.pack_into("<H", ret, 0x0, self.rsp_codes[self.version])
+        return ret

titles/idz/handlers/check_team_names.py

@@ -0,0 +1,20 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerCheckTeamName(IDZHandlerBase):
+    cmd_codes = [0x00A2, 0x00A2, 0x0097, 0x0097]
+    rsp_codes = [0x00A3, 0x00A3, 0x0098, 0x0098]
+    name = "check_team_name"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x0010
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        struct.pack_into("<I", ret, 0x4, 0x1)
+        return data

titles/idz/handlers/create_auto_team.py

@@ -0,0 +1,63 @@
+from operator import indexOf
+import struct
+import json
+from random import choice, randrange
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+AUTO_TEAM_NAMES = ["スピードスターズ", "レッドサンズ", "ナイトキッズ"]
+FULL_WIDTH_NUMS = [
+    "\uff10",
+    "\uff11",
+    "\uff12",
+    "\uff13",
+    "\uff14",
+    "\uff15",
+    "\uff16",
+    "\uff17",
+    "\uff18",
+    "\uff19",
+]
+
+
+class IDZHandlerCreateAutoTeam(IDZHandlerBase):
+    cmd_codes = [0x007B, 0x007B, 0x0077, 0x0077]
+    rsp_codes = [0x007C, 0x007C, 0x0078, 0x0078]
+    name = "create_auto_team"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x0CA0
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        aime_id = struct.unpack_from("<I", data, 0x04)[0]
+        name = choice(AUTO_TEAM_NAMES)
+        bg = indexOf(AUTO_TEAM_NAMES, name)
+        number = (
+            choice(FULL_WIDTH_NUMS) + choice(FULL_WIDTH_NUMS) + choice(FULL_WIDTH_NUMS)
+        )
+
+        tdata = {
+            "id": aime_id,
+            "bg": bg,
+            "fx": 0,
+        }
+
+        tdata = {
+            "id": aime_id,
+            "name": (name + number),
+            "bg": bg,
+            "fx": 0,
+        }
+        tname = tdata["name"].encode("shift-jis")
+
+        struct.pack_into("<I", ret, 0x0C, tdata["id"])
+        struct.pack_into(f"{len(tname)}s", ret, 0x24, tname)
+        struct.pack_into("<I", ret, 0x80, tdata["id"])
+        struct.pack_into(f"<I", ret, 0xD8, tdata["bg"])
+        struct.pack_into(f"<I", ret, 0xDC, tdata["fx"])
+
+        return ret

titles/idz/handlers/create_profile.py

@@ -0,0 +1,73 @@
+import json
+import struct
+from datetime import datetime
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerCreateProfile(IDZHandlerBase):
+    cmd_codes = [0x0066, 0x0066, 0x0064, 0x0064]
+    rsp_codes = [0x0067, 0x0065, 0x0065, 0x0065]
+    name = "create_profile"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x0020
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+
+        aime_id = struct.unpack_from("<L", data, 0x04)[0]
+        name = data[0x1E:0x0034].decode("shift-jis").replace("\x00", "")
+        car = data[0x40:0xA0].hex()
+        chara = data[0xA8:0xBC].hex()
+
+        self.logger.info(f"Create profile for {name} (aime id {aime_id})")
+
+        auto_team = None
+        if not auto_team:
+            team = {"bg": 0, "id": 0, "shop": ""}
+        else:
+            tdata = json.loads(auto_team["data"])
+
+            team = {"bg": tdata["bg"], "id": tdata["fx"], "shop": ""}
+
+        profile_data = {
+            "profile": {
+                "xp": 0,
+                "lv": 1,
+                "fame": 0,
+                "dpoint": 0,
+                "milage": 0,
+                "playstamps": 0,
+                "last_login": int(datetime.now().timestamp()),
+                "car_str": car,  # These should probably be chaged to dicts
+                "chara_str": chara,  # But this works for now...
+            },
+            "options": {
+                "music": 0,
+                "pack": 13640,
+                "aura": 0,
+                "paper_cup": 0,
+                "gauges": 5,
+                "driving_style": 0,
+            },
+            "missions": {"team": [], "solo": []},
+            "story": {"x": 0, "y": 0, "rows": {}},
+            "unlocks": {
+                "auras": 1,
+                "cup": 0,
+                "gauges": 32,
+                "music": 0,
+                "last_mileage_reward": 0,
+            },
+            "team": team,
+        }
+
+        if self.version > 2:
+            struct.pack_into("<L", ret, 0x04, aime_id)
+        else:
+            struct.pack_into("<L", ret, 0x08, aime_id)
+        return ret

titles/idz/handlers/discover_profile.py

@@ -0,0 +1,24 @@
+import struct
+from typing import Tuple, List, Dict
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerDiscoverProfile(IDZHandlerBase):
+    cmd_codes = [0x006B, 0x0067]
+    rsp_codes = [0x006C, 0x0068, 0x0068, 0x0068]
+    name = "discover_profile"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x0010
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        user_id = struct.unpack_from("<I", data, 0x04)[0]
+        profile = None
+
+        struct.pack_into("<H", ret, 0x04, int(profile is not None))
+        return ret

titles/idz/handlers/load_2on2.py

@@ -0,0 +1,38 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerLoad2on2A(IDZHandlerBase):
+    cmd_codes = [0x00B0, 0x00B0, 0x00A3, 0x00A3]
+    rsp_codes = [0x00B1, 0x00B1, 0x00A4, 0x00A4]
+    name = "load_2on2A"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x04C0
+
+        if version >= IDZConstants.VER_IDZ_210:
+            self.size = 0x1290
+
+    def handle(self, data: bytes) -> bytearray:
+        return super().handle(data)
+
+
+class IDZHandlerLoad2on2B(IDZHandlerBase):
+    cmd_codes = [0x0132] * 4
+    rsp_codes = [0x0133] * 4
+    name = "load_2on2B"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x04C0
+
+        if version >= IDZConstants.VER_IDZ_210:
+            self.size = 0x0540
+
+    def handle(self, data: bytes) -> bytearray:
+        return super().handle(data)

titles/idz/handlers/load_config.py

@@ -0,0 +1,43 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerLoadConfigA(IDZHandlerBase):
+    cmd_codes = [0x0004] * IDZConstants.NUM_VERS
+    rsp_codes = [0x0005] * IDZConstants.NUM_VERS
+    name = "load_config_a"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x01A0
+
+        if self.version > 1:
+            self.size = 0x05E0
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        struct.pack_into("<H", ret, 0x02, 1)
+        struct.pack_into("<I", ret, 0x16, 230)
+        return ret
+
+
+class IDZHandlerLoadConfigB(IDZHandlerBase):
+    cmd_codes = [0x00AB, 0x00AB, 0x00A0, 0x00A0]
+    rsp_codes = [0x00AC, 0x00AC, 0x00A1, 0x00A1]
+    name = "load_config_b"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x0230
+
+        if self.version > 1:
+            self.size = 0x0240
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        struct.pack_into("<H", ret, 0x02, 1)
+        return ret

titles/idz/handlers/load_ghost.py

@@ -0,0 +1,40 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerLoadGhost(IDZHandlerBase):
+    cmd_codes = [0x00A0, 0x00A0, 0x0095, 0x0095]
+    rsp_codes = [0x00A1, 0x00A1, 0x0096, 0x0096]
+    name = "load_ghost"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x0070
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        struct.pack_into("<I", ret, 0x02, 0x5)
+
+        struct.pack_into("<L", ret, 0x04, 0x0)
+        struct.pack_into("<l", ret, 0x08, -1)
+        struct.pack_into("<L", ret, 0x0C, 0x1D4C0)
+        struct.pack_into("<L", ret, 0x10, 0x1D4C0)
+        struct.pack_into("<L", ret, 0x14, 0x1D4C0)
+
+        struct.pack_into("<L", ret, 0x38, 0x0)
+        struct.pack_into("<l", ret, 0x3C, -1)
+        struct.pack_into("<L", ret, 0x40, 0x1D4C0)
+        struct.pack_into("<L", ret, 0x44, 0x1D4C0)
+        struct.pack_into("<L", ret, 0x48, 0x1D4C0)
+
+        struct.pack_into("<L", ret, 0x4C, 0x1D4C0)
+        struct.pack_into("<i", ret, 0x50, -1)
+        struct.pack_into("<H", ret, 0x52, 0)
+        struct.pack_into("<H", ret, 0x53, 0)
+        struct.pack_into("<H", ret, 0x54, 0)
+        struct.pack_into("<H", ret, 0x55, 0)
+        struct.pack_into("<H", ret, 0x58, 0)
+        return ret

titles/idz/handlers/load_profile.py

@@ -0,0 +1,29 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerLoadProfile(IDZHandlerBase):
+    cmd_codes = [0x0067, 0x012F, 0x012F, 0x0142]
+    rsp_codes = [0x0065, 0x012E, 0x012E, 0x0141]
+    name = "load_profile"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+
+        if self.version == IDZConstants.VER_IDZ_110:
+            self.size = 0x0D30
+        elif self.version == IDZConstants.VER_IDZ_130:
+            self.size = 0x0EA0
+        elif self.version == IDZConstants.VER_IDZ_210:
+            self.size = 0x1360
+        elif self.version == IDZConstants.VER_IDZ_230:
+            self.size = 0x1640
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        aime_id = struct.unpack_from("<L", data, 0x04)[0]
+        return ret

titles/idz/handlers/load_reward_table.py

@@ -0,0 +1,18 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerLoadRewardTable(IDZHandlerBase):
+    cmd_codes = [0x0086, 0x0086, 0x007F, 0x007F]
+    rsp_codes = [0x0087, 0x0087, 0x0080, 0x0080]
+    name = "load_reward_table"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x01C0
+
+    def handle(self, data: bytes) -> bytearray:
+        return super().handle(data)

titles/idz/handlers/load_server_info.py

@@ -0,0 +1,96 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerLoadServerInfo(IDZHandlerBase):
+    cmd_codes = [0x0006] * IDZConstants.NUM_VERS
+    rsp_codes = [0x0007] * IDZConstants.NUM_VERS
+    name = "load_server_info1"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x04B0
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        offset = 0
+        if self.version >= IDZConstants.VER_IDZ_210:
+            offset = 2
+
+        news_str = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDDF/230/news/news80**.txt"
+        err_str = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDDF/230/error"
+
+        len_hostname = len(self.core_config.title.hostname)
+        len_news = len(news_str)
+        len_error = len(err_str)
+
+        struct.pack_into("<I", ret, 0x2 + offset, 1)  # Status
+        struct.pack_into(
+            f"{len_hostname}s",
+            ret,
+            0x4 + offset,
+            self.core_config.title.hostname.encode(),
+        )
+        struct.pack_into("<I", ret, 0x84 + offset, self.game_cfg.ports.userdb)
+        struct.pack_into("<I", ret, 0x86 + offset, self.game_cfg.ports.userdb + 1)
+
+        struct.pack_into(
+            f"{len_hostname}s",
+            ret,
+            0x88 + offset,
+            self.core_config.title.hostname.encode(),
+        )
+        struct.pack_into("<I", ret, 0x108 + offset, self.game_cfg.ports.match - 1)
+        struct.pack_into("<I", ret, 0x10A + offset, self.game_cfg.ports.match - 3)
+        struct.pack_into("<I", ret, 0x10C + offset, self.game_cfg.ports.match - 2)
+
+        struct.pack_into("<I", ret, 0x10E + offset, self.game_cfg.ports.match + 2)
+        struct.pack_into("<I", ret, 0x110 + offset, self.game_cfg.ports.match + 3)
+        struct.pack_into("<I", ret, 0x112 + offset, self.game_cfg.ports.match + 1)
+
+        struct.pack_into(
+            f"{len_hostname}s",
+            ret,
+            0x114 + offset,
+            self.core_config.title.hostname.encode(),
+        )
+        struct.pack_into("<I", ret, 0x194 + offset, self.game_cfg.ports.echo + 2)
+
+        struct.pack_into(
+            f"{len_hostname}s",
+            ret,
+            0x0199 + offset,
+            self.core_config.title.hostname.encode(),
+        )
+        struct.pack_into("<I", ret, 0x0219 + offset, self.game_cfg.ports.echo + 3)
+
+        struct.pack_into(
+            f"{len_hostname}s",
+            ret,
+            0x021C + offset,
+            self.core_config.title.hostname.encode(),
+        )
+        struct.pack_into(
+            f"{len_hostname}s",
+            ret,
+            0x029C + offset,
+            self.core_config.title.hostname.encode(),
+        )
+        struct.pack_into(
+            f"{len_hostname}s",
+            ret,
+            0x031C + offset,
+            self.core_config.title.hostname.encode(),
+        )
+
+        struct.pack_into("<I", ret, 0x39C + offset, self.game_cfg.ports.echo)
+        struct.pack_into("<I", ret, 0x39E + offset, self.game_cfg.ports.echo + 1)
+
+        struct.pack_into(f"{len_news}s", ret, 0x03A0 + offset, news_str.encode())
+        struct.pack_into(f"{len_error}s", ret, 0x0424 + offset, err_str.encode())
+
+        return ret

titles/idz/handlers/load_team_ranking.py

@@ -0,0 +1,31 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerLoadTeamRankingA(IDZHandlerBase):
+    cmd_codes = [0x00B9, 0x00B9, 0x00A7, 0x00A7]
+    rsp_codes = [0x00B1] * 4
+    name = "load_team_ranking_a"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x0BA0
+
+    def handle(self, data: bytes) -> bytearray:
+        return super().handle(data)
+
+
+class IDZHandlerLoadTeamRankingB(IDZHandlerBase):
+    cmd_codes = [0x00BB, 0x00BB, 0x00A9, 0x00A9]
+    rsp_codes = [0x00A8] * 4
+    name = "load_team_ranking_b"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x0BA0
+
+    def handle(self, data: bytes) -> bytearray:
+        return super().handle(data)

titles/idz/handlers/load_top_ten.py

@@ -0,0 +1,35 @@
+import struct
+from typing import Tuple, List, Dict
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerLoadTopTen(IDZHandlerBase):
+    cmd_codes = [0x012C] * 4
+    rsp_codes = [0x00CE] * 4
+    name = "load_top_ten"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x1720
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        tracks_dates: List[Tuple[int, int]] = []
+        for i in range(32):
+            tracks_dates.append(
+                (
+                    struct.unpack_from("<H", data, 0x04 + (2 * i))[0],
+                    "little",
+                    struct.unpack_from("<I", data, 0x44 + (4 * i))[0],
+                    "little",
+                )
+            )
+        # TODO: Best scores
+        for i in range(3):
+            offset = 0x16C0 + 0x1C * i
+            struct.pack_into("<B", ret, offset + 0x02, 0xFF)
+
+        return ret

titles/idz/handlers/lock_profile.py

@@ -0,0 +1,42 @@
+import struct
+from typing import Dict
+from datetime import datetime, timedelta
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerLockProfile(IDZHandlerBase):
+    cmd_codes = [0x0069, 0x0069, 0x0065, 0x0065]
+    rsp_codes = [0x006A, 0x006A, 0x0066, 0x0066]
+    name = "lock_profile"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x0020
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        profile_data = {
+            "status": IDZConstants.PROFILE_STATUS.UNLOCKED.value,
+            "expire_time": int(
+                (datetime.now() + timedelta(hours=1)).timestamp() / 1000
+            ),
+        }
+        user_id = struct.unpack_from("<I", data, 0x04)[0]
+        profile = None
+
+        if profile is None and self.version > 0:
+            old_profile = None
+            if old_profile is not None:
+                profile_data["status"] = IDZConstants.PROFILE_STATUS.OLD.value
+
+        return self.handle_common(profile_data, ret)
+
+    def handle_common(cls, data: Dict, ret: bytearray) -> bytearray:
+        struct.pack_into("<B", ret, 0x18, data["status"])
+        struct.pack_into("<h", ret, 0x1A, -1)
+        struct.pack_into("<I", ret, 0x1C, data["expire_time"])
+        return ret

titles/idz/handlers/save_expedition.py

@@ -0,0 +1,15 @@
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerSaveExpedition(IDZHandlerBase):
+    cmd_codes = [0x008C, 0x013F]
+    name = "save_expedition"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+
+    def handle(self, data: bytes) -> bytearray:
+        return super().handle(data)

titles/idz/handlers/save_profile.py

@@ -0,0 +1,16 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerSaveProfile(IDZHandlerBase):
+    cmd_codes = [0x0068, 0x0138, 0x0138, 0x0143]
+    name = "save_profile"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+
+    def handle(self, data: bytes) -> bytearray:
+        return super().handle(data)

titles/idz/handlers/save_time_attack.py

@@ -0,0 +1,23 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerSaveTimeAttack(IDZHandlerBase):
+    cmd_codes = [0x00CD, 0x0136, 0x0136, 0x0136]
+    rsp_codes = [0x00CE, 0x00CE, 0x00CD, 0x00CD]
+    name = "save_time_attack"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x00B0
+
+        if self.version > IDZConstants.VER_IDZ_130:
+            self.size = 0x00F0
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        return ret

titles/idz/handlers/save_topic.py

@@ -0,0 +1,18 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerSaveTopic(IDZHandlerBase):
+    cmd_codes = [0x009A, 0x009A, 0x0091, 0x0091]
+    rsp_codes = [0x009B, 0x009B, 0x0092, 0x0092]
+    name = "save_topic"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x05D0
+
+    def handle(self, data: bytes) -> bytearray:
+        return super().handle(data)

titles/idz/handlers/unknown.py

@@ -0,0 +1,13 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerUnknown(IDZHandlerBase):
+    cmd_codes = [0x00AD, 0x00AD, 0x00A2, 0x00A2]
+    name = "unknown"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)

titles/idz/handlers/unlock_profile.py

@@ -0,0 +1,21 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerUnlockProfile(IDZHandlerBase):
+    cmd_codes = [0x006F, 0x006F, 0x006B, 0x006B]
+    rsp_codes = [0x0070, 0x0070, 0x006C, 0x006C]
+    name = "unlock_profile"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x0010
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        struct.pack_into("<H", ret, 0x4, 1)
+        return ret

titles/idz/handlers/update_provisional_store_rank.py

@@ -0,0 +1,21 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+
+
+class IDZHandlerUpdateProvisionalStoreRank(IDZHandlerBase):
+    cmd_codes = [0x0082, 0x0082, 0x007C, 0x007C]
+    rsp_codes = [0x0083, 0x0083, 0x007D, 0x007D]
+    name = "update_provisional_store_ranking"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+        self.size = 0x02B0
+
+    def handle(self, data: bytes) -> bytearray:
+        return super().handle(data)
+
+    def handle_common(cls, aime_id: int, ret: bytearray) -> bytearray:
+        pass

titles/idz/handlers/update_story_clear_num.py

@@ -0,0 +1,27 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandlerUpdateStoryClearNum(IDZHandlerBase):
+    cmd_codes = [0x007F, 0x097F, 0x013D, 0x0144]
+    rsp_codes = [0x0080, 0x013E, 0x013E, 0x0145]
+    name = "update_story_clear_num"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+
+        if self.version == IDZConstants.VER_IDZ_110:
+            self.size = 0x0220
+        elif self.version == IDZConstants.VER_IDZ_130:
+            self.size = 0x04F0
+        elif self.version == IDZConstants.VER_IDZ_210:
+            self.size = 0x0510
+        elif self.version == IDZConstants.VER_IDZ_230:
+            self.size = 0x0800
+
+    def handle(self, data: bytes) -> bytearray:
+        return super().handle(data)

titles/idz/handlers/update_team_points.py

@@ -0,0 +1,18 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandleUpdateTeamPoints(IDZHandlerBase):
+    cmd_codes = [0x0081, 0x0081, 0x007B, 0x007B]
+    name = "unlock_profile"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        return ret

titles/idz/handlers/update_ui_report.py

@@ -0,0 +1,18 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandleUpdateUIReport(IDZHandlerBase):
+    cmd_codes = [0x0084, 0x0084, 0x007E, 0x007E]
+    name = "update_ui_report"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        return ret

titles/idz/handlers/update_user_log.py

@@ -0,0 +1,18 @@
+import struct
+
+from .base import IDZHandlerBase
+from core.config import CoreConfig
+from ..config import IDZConfig
+from ..const import IDZConstants
+
+
+class IDZHandleUpdateUserLog(IDZHandlerBase):
+    cmd_codes = [0x00BD, 0x00BD, 0x00AB, 0x00B3]
+    name = "update_user_log"
+
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig, version: int) -> None:
+        super().__init__(core_cfg, game_cfg, version)
+
+    def handle(self, data: bytes) -> bytearray:
+        ret = super().handle(data)
+        return ret

titles/idz/index.py

@@ -0,0 +1,174 @@
+from twisted.web.http import Request
+import yaml
+import logging
+import coloredlogs
+from logging.handlers import TimedRotatingFileHandler
+from os import path
+from typing import Tuple, List
+from twisted.internet import reactor, endpoints
+from twisted.web import server, resource
+import importlib
+
+from core.config import CoreConfig
+from .config import IDZConfig
+from .const import IDZConstants
+from .userdb import IDZUserDBFactory, IDZUserDBWeb, IDZKey
+from .echo import IDZEcho
+from .handlers import IDZHandlerLoadConfigB
+
+
+class IDZServlet:
+    def __init__(self, core_cfg: CoreConfig, cfg_dir: str) -> None:
+        self.core_cfg = core_cfg
+        self.game_cfg = IDZConfig()
+        if path.exists(f"{cfg_dir}/{IDZConstants.CONFIG_NAME}"):
+            self.game_cfg.update(
+                yaml.safe_load(open(f"{cfg_dir}/{IDZConstants.CONFIG_NAME}"))
+            )
+
+        self.logger = logging.getLogger("idz")
+        if not hasattr(self.logger, "inited"):
+            log_fmt_str = "[%(asctime)s] IDZ | %(levelname)s | %(message)s"
+            log_fmt = logging.Formatter(log_fmt_str)
+            fileHandler = TimedRotatingFileHandler(
+                "{0}/{1}.log".format(self.core_cfg.server.log_dir, "idz"),
+                encoding="utf8",
+                when="d",
+                backupCount=10,
+            )
+
+            self.rsa_keys: List[IDZKey] = []
+
+            fileHandler.setFormatter(log_fmt)
+
+            consoleHandler = logging.StreamHandler()
+            consoleHandler.setFormatter(log_fmt)
+
+            self.logger.addHandler(fileHandler)
+            self.logger.addHandler(consoleHandler)
+
+            self.logger.setLevel(self.game_cfg.server.loglevel)
+            coloredlogs.install(
+                level=self.game_cfg.server.loglevel, logger=self.logger, fmt=log_fmt_str
+            )
+            self.logger.inited = True
+
+    @classmethod
+    def rsaHashKeyN(cls, data):
+        hash_ = 0
+        for i in data:
+            hash_ = (
+                hash_ * IDZConstants.HASH_MUL + (i ^ IDZConstants.HASH_XOR)
+                ^ IDZConstants.HASH_LUT[i & 0xF]
+            )
+            hash_ &= 0xFFFFFFFF
+        return hash_
+
+    @classmethod
+    def get_allnet_info(
+        cls, game_code: str, core_cfg: CoreConfig, cfg_dir: str
+    ) -> Tuple[bool, str, str]:
+        game_cfg = IDZConfig()
+        if path.exists(f"{cfg_dir}/{IDZConstants.CONFIG_NAME}"):
+            game_cfg.update(
+                yaml.safe_load(open(f"{cfg_dir}/{IDZConstants.CONFIG_NAME}"))
+            )
+
+        if not game_cfg.server.enable:
+            return (False, "", "")
+
+        if len(game_cfg.rsa_keys) <= 0 or not game_cfg.server.aes_key:
+            logging.getLogger("idz").error("IDZ: No RSA/AES  keys! IDZ cannot start")
+            return (False, "", "")
+
+        hostname = (
+            core_cfg.title.hostname
+            if not game_cfg.server.hostname
+            else game_cfg.server.hostname
+        )
+        return (
+            True,
+            f"",
+            f"{hostname}:{game_cfg.ports.userdb}",
+        )
+
+    def setup(self):
+        for key in self.game_cfg.rsa_keys:
+            if "N" not in key or "d" not in key or "e" not in key:
+                self.logger.error(f"Invalid IDZ key {key}")
+                continue
+
+            hashN = self.rsaHashKeyN(str(key["N"]).encode())
+            self.rsa_keys.append(IDZKey(key["N"], key["d"], key["e"], hashN))
+
+        if len(self.rsa_keys) <= 0:
+            self.logger.error("No valid RSA keys provided! IDZ cannot start!")
+            return
+
+        handler_map = [{} for _ in range(IDZConstants.NUM_VERS)]
+        handler_mod = mod = importlib.import_module(f"titles.idz.handlers")
+
+        for cls_name in dir(handler_mod):
+            if cls_name.startswith("__"):
+                continue
+
+            try:
+                mod = getattr(handler_mod, cls_name)
+                mod_cmds: List = getattr(mod, "cmd_codes")
+                while len(mod_cmds) < IDZConstants.NUM_VERS:
+                    mod_cmds.append(None)
+
+                for i in range(len(mod_cmds)):
+                    if mod_cmds[i] is None:
+                        mod_cmds[i] = mod_cmds[i - 1]
+
+                    handler_map[i][mod_cmds[i]] = mod
+
+            except AttributeError as e:
+                continue
+
+        endpoints.serverFromString(
+            reactor,
+            f"tcp:{self.game_cfg.ports.userdb}:interface={self.core_cfg.server.listen_address}",
+        ).listen(
+            IDZUserDBFactory(self.core_cfg, self.game_cfg, self.rsa_keys, handler_map)
+        )
+
+        reactor.listenUDP(
+            self.game_cfg.ports.echo, IDZEcho(self.core_cfg, self.game_cfg)
+        )
+        reactor.listenUDP(
+            self.game_cfg.ports.echo + 1, IDZEcho(self.core_cfg, self.game_cfg)
+        )
+        reactor.listenUDP(
+            self.game_cfg.ports.match, IDZEcho(self.core_cfg, self.game_cfg)
+        )
+        reactor.listenUDP(
+            self.game_cfg.ports.userdb + 1, IDZEcho(self.core_cfg, self.game_cfg)
+        )
+
+        self.logger.info(f"UserDB Listening on port {self.game_cfg.ports.userdb}")
+
+    def render_POST(self, request: Request, version: int, url_path: str) -> bytes:
+        req_raw = request.content.getvalue()
+        self.logger.info(f"IDZ POST request: {url_path} - {req_raw}")
+        return b""
+
+    def render_GET(self, request: Request, version: int, url_path: str) -> bytes:
+        self.logger.info(f"IDZ GET request: {url_path}")
+        request.responseHeaders.setRawHeaders(
+            "Content-Type", [b"text/plain; charset=utf-8"]
+        )
+        request.responseHeaders.setRawHeaders(
+            "Last-Modified", [b"Sun, 23 Apr 2023 05:33:20 GMT"]
+        )
+
+        news = (
+            self.game_cfg.server.news
+            if self.game_cfg.server.news
+            else f"Welcome to Initial D Arcade Stage Zero on {self.core_cfg.server.name}!"
+        )
+        news += "\r\n"
+        news = "1979/01/01 00:00:00 2099/12/31 23:59:59 " + news
+
+        return news.encode()

titles/idz/userdb.py

@@ -0,0 +1,201 @@
+from twisted.internet.protocol import Factory, Protocol
+import logging, coloredlogs
+from Crypto.Cipher import AES
+import struct
+from typing import Dict, Optional, List, Type
+from twisted.web import server, resource
+from twisted.internet import reactor, endpoints
+from twisted.web.http import Request
+from routes import Mapper
+import random
+from os import walk
+import importlib
+
+from core.config import CoreConfig
+from .database import IDZData
+from .config import IDZConfig
+from .const import IDZConstants
+from .handlers import IDZHandlerBase
+
+HANDLER_MAP: List[Dict]
+
+
+class IDZKey:
+    def __init__(self, n, d, e, hashN: int) -> None:
+        self.N = n
+        self.d = d
+        self.e = e
+        self.hashN = hashN
+
+
+class IDZUserDBProtocol(Protocol):
+    def __init__(
+        self,
+        core_cfg: CoreConfig,
+        game_cfg: IDZConfig,
+        keys: List[IDZKey],
+        handlers: List[Dict],
+    ) -> None:
+        self.logger = logging.getLogger("idz")
+        self.core_config = core_cfg
+        self.game_config = game_cfg
+        self.rsa_keys = keys
+        self.handlers = handlers
+        self.static_key = bytes.fromhex(self.game_config.server.aes_key)
+        self.version = None
+        self.version_internal = None
+        self.skip_next = False
+
+    def append_padding(self, data: bytes):
+        """Appends 0s to the end of the data until it's at the correct size"""
+        length = struct.unpack_from("<H", data, 6)
+        padding_size = length[0] - len(data)
+        data += bytes(padding_size)
+        return data
+
+    def connectionMade(self) -> None:
+        self.logger.debug(f"{self.transport.getPeer().host} Connected")
+        base = 0
+
+        for i in range(len(self.static_key) - 1):
+            shift = 8 * i
+            byte = self.static_key[i]
+
+            base |= byte << shift
+
+        rsa_key = random.choice(self.rsa_keys)
+        key_enc: int = pow(base, rsa_key.e, rsa_key.N)
+        result = (
+            key_enc.to_bytes(0x40, "little")
+            + struct.pack("<I", 0x01020304)
+            + rsa_key.hashN.to_bytes(4, "little")
+        )
+
+        self.logger.debug(f"Send handshake {result.hex()}")
+
+        self.transport.write(result)
+
+    def connectionLost(self, reason) -> None:
+        self.logger.debug(
+            f"{self.transport.getPeer().host} Disconnected - {reason.value}"
+        )
+
+    def dataReceived(self, data: bytes) -> None:
+        self.logger.debug(f"Receive data {data.hex()}")
+        crypt = AES.new(self.static_key, AES.MODE_ECB)
+        
+        try:
+            data_dec = crypt.decrypt(data)
+        
+        except Exception as e:
+            self.logger.error(f"Failed to decrypt UserDB request from {self.transport.getPeer().host} because {e} - {data.hex()}")
+        
+        self.logger.debug(f"Decrypt data {data_dec.hex()}")
+
+        magic = struct.unpack_from("<I", data_dec, 0)[0]
+
+        if magic == 0xFE78571D:
+            # Ignore
+            self.logger.info(f"Userdb serverbox request {data_dec.hex()}")
+            self.skip_next = True
+
+            self.transport.write(b"\x00")
+            return
+
+        elif magic == 0x01020304:
+            self.version = int(data_dec[16:19].decode())
+
+            if self.version == 110:
+                self.version_internal = IDZConstants.VER_IDZ_110
+            elif self.version == 130:
+                self.version_internal = IDZConstants.VER_IDZ_130
+            elif self.version == 210:
+                self.version_internal = IDZConstants.VER_IDZ_210
+            elif self.version == 230:
+                self.version_internal = IDZConstants.VER_IDZ_230
+            else:
+                self.logger.warn(f"Bad version v{self.version}")
+                self.version = None
+                self.version_internal = None
+
+            self.logger.debug(
+                f"Userdb v{self.version} handshake response from {self.transport.getPeer().host}"
+            )
+            return
+
+        elif self.skip_next:
+            self.skip_next = False
+            self.transport.write(b"\x00")
+            return
+
+        elif self.version is None:
+            # We didn't get a handshake before, and this isn't one now, so we're up the creek
+            self.logger.info(
+                f"Bad UserDB request from from {self.transport.getPeer().host}"
+            )
+            self.transport.write(b"\x00")
+            return
+
+        cmd = struct.unpack_from("<H", data_dec, 0)[0]
+
+        handler_cls: Optional[Type[IDZHandlerBase]] = self.handlers[
+            self.version_internal
+        ].get(cmd, None)
+        if handler_cls is None:
+            self.logger.warn(f"No handler for v{self.version} {hex(cmd)} cmd")
+            handler_cls = IDZHandlerBase
+
+        handler = handler_cls(self.core_config, self.game_config, self.version_internal)
+        self.logger.info(
+            f"Userdb v{self.version} {handler.name} request from {self.transport.getPeer().host}"
+        )
+        response = handler.handle(data_dec)
+
+        self.logger.debug(f"Response: {response.hex()}")
+
+        crypt = AES.new(self.static_key, AES.MODE_ECB)
+        response_enc = crypt.encrypt(response)
+
+        self.transport.write(response_enc)
+
+
+class IDZUserDBFactory(Factory):
+    protocol = IDZUserDBProtocol
+
+    def __init__(
+        self,
+        cfg: CoreConfig,
+        game_cfg: IDZConfig,
+        keys: List[IDZKey],
+        handlers: List[Dict],
+    ) -> None:
+        self.core_config = cfg
+        self.game_config = game_cfg
+        self.keys = keys
+        self.handlers = handlers
+
+    def buildProtocol(self, addr):
+        return IDZUserDBProtocol(
+            self.core_config, self.game_config, self.keys, self.handlers
+        )
+
+
+class IDZUserDBWeb(resource.Resource):
+    def __init__(self, core_cfg: CoreConfig, game_cfg: IDZConfig):
+        super().__init__()
+        self.isLeaf = True
+        self.core_config = core_cfg
+        self.game_config = game_cfg
+        self.logger = logging.getLogger("idz")
+
+    def render_POST(self, request: Request) -> bytes:
+        self.logger.info(
+            f"IDZUserDBWeb POST from {request.getClientAddress().host} to {request.uri} with data {request.content.getvalue()}"
+        )
+        return b""
+
+    def render_GET(self, request: Request) -> bytes:
+        self.logger.info(
+            f"IDZUserDBWeb GET from {request.getClientAddress().host} to {request.uri}"
+        )
+        return b""

titles/mai2/__init__.py

@@ -7,4 +7,4 @@ index = Mai2Servlet
 database = Mai2Data
 reader = Mai2Reader
 game_codes = [Mai2Constants.GAME_CODE]
-current_schema_version = 4
+current_schema_version = 5

titles/mai2/base.py

@@ -17,7 +17,14 @@ class Mai2Base:
         self.data = Mai2Data(cfg)
         self.logger = logging.getLogger("mai2")
         
+        if self.core_config.server.is_develop and self.core_config.title.port > 0:
+            self.old_server = f"http://{self.core_config.title.hostname}:{self.core_config.title.port}/SDEY/100/"
+        
+        else:
+            self.old_server = f"http://{self.core_config.title.hostname}/SDEY/100/"
+
     def handle_get_game_setting_api_request(self, data: Dict):
+        # TODO: See if making this epoch 0 breaks things
         reboot_start = date.strftime(
             datetime.now() + timedelta(hours=3), Mai2Constants.DATE_TIME_FORMAT
         )
@@ -34,7 +41,7 @@ class Mai2Base:
                 "movieStatus": 0,
                 "movieServerUri": "",
                 "deliverServerUri": "",
-                "oldServerUri": "",
+                "oldServerUri": self.old_server,
                 "usbDlServerUri": "",
                 "rebootInterval": 0,
             },
@@ -300,7 +307,9 @@ class Mai2Base:
         ):
             for fsr in upsert["userFriendSeasonRankingList"]:
                 fsr["recordDate"] = (
-                    datetime.strptime(fsr["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"),
+                    datetime.strptime(
+                        fsr["recordDate"], f"{Mai2Constants.DATE_TIME_FORMAT}.0"
+                    ),
                 )
                 self.data.item.put_friend_season_ranking(user_id, fsr)
 

titles/mai2/const.py

@@ -37,9 +37,9 @@ class Mai2Constants:
         "maimai DX PLUS",
         "maimai DX Splash",
         "maimai DX Splash PLUS",
-        "maimai DX Universe",
+        "maimai DX UNiVERSE",
-        "maimai DX Universe PLUS",
+        "maimai DX UNiVERSE PLUS",
-        "maimai DX Festival"
+        "maimai DX FESTiVAL",
     )
 
     @classmethod

titles/mai2/index.py

@@ -38,7 +38,7 @@ class Mai2Servlet:
             Mai2SplashPlus,
             Mai2Universe,
             Mai2UniversePlus,
-            Mai2Festival
+            Mai2Festival,
         ]
 
         self.logger = logging.getLogger("mai2")
@@ -82,13 +82,13 @@ class Mai2Servlet:
             return (
                 True,
                 f"http://{core_cfg.title.hostname}:{core_cfg.title.port}/{game_code}/$v/",
-                f"{core_cfg.title.hostname}:{core_cfg.title.port}/",
+                f"{core_cfg.title.hostname}:{core_cfg.title.port}",
             )
 
         return (
             True,
             f"http://{core_cfg.title.hostname}/{game_code}/$v/",
-            f"{core_cfg.title.hostname}/",
+            f"{core_cfg.title.hostname}",
         )
 
     def render_POST(self, request: Request, version: int, url_path: str) -> bytes:
@@ -134,9 +134,7 @@ class Mai2Servlet:
 
         req_data = json.loads(unzip)
 
-        self.logger.info(
+        self.logger.info(f"v{version} {endpoint} request from {client_ip}")
-            f"v{version} {endpoint} request from {client_ip}"
-        )
         self.logger.debug(req_data)
 
         func_to_find = "handle_" + inflection.underscore(endpoint) + "_request"

titles/mai2/schema/item.py

@@ -39,8 +39,8 @@ card = Table(
     Column("cardTypeId", Integer, nullable=False),
     Column("charaId", Integer, nullable=False),
     Column("mapId", Integer, nullable=False),
-    Column("startDate", TIMESTAMP, server_default="2018-01-01 00:00:00.0"),
+    Column("startDate", TIMESTAMP, nullable=False, server_default=func.now()),
-    Column("endDate", TIMESTAMP, server_default="2038-01-01 00:00:00.0"),
+    Column("endDate", TIMESTAMP, nullable=False),
     UniqueConstraint("user", "cardId", "cardTypeId", name="mai2_item_card_uk"),
     mysql_charset="utf8mb4",
 )
@@ -402,7 +402,7 @@ class Mai2ItemData(BaseData):
         if result is None:
             self.logger.warn(
                 f"put_friend_season_ranking: failed to insert",
-                f"friend_season_ranking! aime_id: {aime_id}"
+                f"friend_season_ranking! aime_id: {aime_id}",
             )
             return None
         return result.lastrowid
@@ -444,6 +444,8 @@ class Mai2ItemData(BaseData):
         card_kind: int,
         chara_id: int,
         map_id: int,
+        start_date: datetime,
+        end_date: datetime,
     ) -> Optional[Row]:
         sql = insert(card).values(
             user=user_id,
@@ -451,14 +453,18 @@ class Mai2ItemData(BaseData):
             cardTypeId=card_kind,
             charaId=chara_id,
             mapId=map_id,
+            startDate=start_date,
+            endDate=end_date,
         )
 
-        conflict = sql.on_duplicate_key_update(charaId=chara_id, mapId=map_id)
+        conflict = sql.on_duplicate_key_update(
+            charaId=chara_id, mapId=map_id, startDate=start_date, endDate=end_date
+        )
 
         result = self.execute(conflict)
         if result is None:
             self.logger.warn(
-                f"put_card: failed to insert card! user_id: {user_id}, kind: {kind}"
+                f"put_card: failed to insert card! user_id: {user_id}, kind: {card_kind}"
             )
             return None
         return result.lastrowid

titles/mai2/schema/profile.py

@@ -299,9 +299,11 @@ class Mai2ProfileData(BaseData):
         return result.lastrowid
 
     def get_profile_detail(self, user_id: int, version: int) -> Optional[Row]:
-        sql = select(detail).where(
+        sql = (
-            and_(detail.c.user == user_id, detail.c.version <= version)
+            select(detail)
-        ).order_by(detail.c.version.desc())
+            .where(and_(detail.c.user == user_id, detail.c.version <= version))
+            .order_by(detail.c.version.desc())
+        )
 
         result = self.execute(sql)
         if result is None:
@@ -324,9 +326,11 @@ class Mai2ProfileData(BaseData):
         return result.lastrowid
 
     def get_profile_ghost(self, user_id: int, version: int) -> Optional[Row]:
-        sql = select(ghost).where(
+        sql = (
-            and_(ghost.c.user == user_id, ghost.c.version_int <= version)
+            select(ghost)
-        ).order_by(ghost.c.version.desc())
+            .where(and_(ghost.c.user == user_id, ghost.c.version_int <= version))
+            .order_by(ghost.c.version.desc())
+        )
 
         result = self.execute(sql)
         if result is None:
@@ -349,9 +353,11 @@ class Mai2ProfileData(BaseData):
         return result.lastrowid
 
     def get_profile_extend(self, user_id: int, version: int) -> Optional[Row]:
-        sql = select(extend).where(
+        sql = (
-            and_(extend.c.user == user_id, extend.c.version <= version)
+            select(extend)
-        ).order_by(extend.c.version.desc())
+            .where(and_(extend.c.user == user_id, extend.c.version <= version))
+            .order_by(extend.c.version.desc())
+        )
 
         result = self.execute(sql)
         if result is None:
@@ -374,9 +380,11 @@ class Mai2ProfileData(BaseData):
         return result.lastrowid
 
     def get_profile_option(self, user_id: int, version: int) -> Optional[Row]:
-        sql = select(option).where(
+        sql = (
-            and_(option.c.user == user_id, option.c.version <= version)
+            select(option)
-        ).order_by(option.c.version.desc())
+            .where(and_(option.c.user == user_id, option.c.version <= version))
+            .order_by(option.c.version.desc())
+        )
 
         result = self.execute(sql)
         if result is None:
@@ -399,9 +407,11 @@ class Mai2ProfileData(BaseData):
         return result.lastrowid
 
     def get_profile_rating(self, user_id: int, version: int) -> Optional[Row]:
-        sql = select(rating).where(
+        sql = (
-            and_(rating.c.user == user_id, rating.c.version <= version)
+            select(rating)
-        ).order_by(rating.c.version.desc())
+            .where(and_(rating.c.user == user_id, rating.c.version <= version))
+            .order_by(rating.c.version.desc())
+        )
 
         result = self.execute(sql)
         if result is None:

titles/mai2/universe.py

@@ -104,8 +104,12 @@ class Mai2Universe(Mai2Base):
             tmp.pop("id")
             tmp.pop("user")
 
-            tmp["startDate"] = datetime.strftime(tmp["startDate"], "%Y-%m-%d %H:%M:%S")
+            tmp["startDate"] = datetime.strftime(
-            tmp["endDate"] = datetime.strftime(tmp["endDate"], "%Y-%m-%d %H:%M:%S")
+                tmp["startDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
+            tmp["endDate"] = datetime.strftime(
+                tmp["endDate"], Mai2Constants.DATE_TIME_FORMAT
+            )
             card_list.append(tmp)
 
         return {
@@ -154,6 +158,10 @@ class Mai2Universe(Mai2Base):
         # set a random card serial number
         serial_id = "".join([str(randint(0, 9)) for _ in range(20)])
 
+        # calculate start and end date of the card
+        start_date = datetime.utcnow()
+        end_date = datetime.utcnow() + timedelta(days=15)
+
         user_card = upsert["userCard"]
         self.data.item.put_card(
             user_id,
@@ -161,8 +169,26 @@ class Mai2Universe(Mai2Base):
             user_card["cardTypeId"],
             user_card["charaId"],
             user_card["mapId"],
+            # add the correct start date and also the end date in 15 days
+            start_date,
+            end_date,
         )
 
+        # get the profile extend to save the new bought card
+        extend = self.data.profile.get_profile_extend(user_id, self.version)
+        if extend:
+            extend = extend._asdict()
+            # parse the selectedCardList
+            # 6 = Freedom Pass, 4 = Gold Pass (cardTypeId)
+            selected_cards: list = extend["selectedCardList"]
+
+            # if no pass is already added, add the corresponding pass
+            if not user_card["cardTypeId"] in selected_cards:
+                selected_cards.insert(0, user_card["cardTypeId"])
+
+            extend["selectedCardList"] = selected_cards
+            self.data.profile.put_profile_extend(user_id, self.version, extend)
+
         # properly format userPrintDetail for the database
         upsert.pop("userCard")
         upsert.pop("serialId")
@@ -174,8 +200,8 @@ class Mai2Universe(Mai2Base):
             "returnCode": 1,
             "orderId": 0,
             "serialId": serial_id,
-            "startDate": "2018-01-01 00:00:00",
+            "startDate": datetime.strftime(start_date, Mai2Constants.DATE_TIME_FORMAT),
-            "endDate": "2038-01-01 00:00:00",
+            "endDate": datetime.strftime(end_date, Mai2Constants.DATE_TIME_FORMAT),
         }
 
     def handle_cm_upsert_user_printlog_api_request(self, data: Dict) -> Dict: