fix Word Select mapping across versions

CMakeLists.txt

@@ -106,6 +106,7 @@ add_executable(newserv
     src/Text.cc
     src/TextArchive.cc
     src/Version.cc
+    src/WordSelectTable.cc
 )
 target_include_directories(newserv PUBLIC ${LIBEVENT_INCLUDE_DIR})
 target_link_libraries(newserv phosg ${LIBEVENT_LIBRARIES} pthread)

TODO.md

@@ -12,10 +12,11 @@
 - Build an exception-handling abstraction in ChatCommands that shows formatted error messages in all cases
 - Make reloading happen on separate threads so compression doesn't block active clients
 - Implement decrypt/encrypt actions for VMS files
-- Fix Word Select mapping across versions
 - Make UI strings localizable (e.g. entries in menus, welcome message, etc.)
 - Figure out what causes the corruption message on PC proxy sessions and fix it
 - Enable item tracking in battle/challenge games (everything should already be set up for it to work)
+- Use challenge mode rare tables in challenge mode games (also, apparently it always uses Viridia? verify this)
+- Rewrite REL-based parsers so they don't assume any fixed offsets
 
 ## Episode 3
 

src/CommandFormats.hh

@@ -4607,14 +4607,24 @@ struct G_Unknown_6x73 {
 } __packed__;
 
 // 6x74: Word select
+// There is a bug in PSO GC with regard to this command: the client does not
+// byteswap the header, which means the client_id field is big-endian.
+
+struct WordSelectMessage {
+  le_uint16_t num_tokens;
+  le_uint16_t target_type;
+  parray<le_uint16_t, 8> tokens;
+  le_uint32_t numeric_parameter;
+  le_uint32_t unknown_a4;
+} __packed__;
 
+template <bool IsBigEndian>
 struct G_WordSelect_6x74 {
-  G_ClientIDHeader header;
-  le_uint16_t unknown_a1;
-  le_uint16_t unknown_a2;
-  parray<le_uint16_t, 8> entries;
-  le_uint32_t unknown_a3;
-  le_uint32_t unknown_a4;
+  using U16T = typename std::conditional<IsBigEndian, be_uint16_t, le_uint16_t>::type;
+  uint8_t subcommand;
+  uint8_t size;
+  U16T client_id;
+  WordSelectMessage message;
 } __packed__;
 
 // 6x75: Set quest flag

src/Episode3/DataIndexes.hh

@@ -890,7 +890,7 @@ struct PlayerConfig {
   // This array is updated when a battle is started (via a 6xB4x05 command). The
   // client adds the opposing players' info to ths first two entries here if the
   // opponents are human. (The existing entries are always moved back by two
-  // slots, but if either or both opponents are not humans, one or both of the
+  // slots, but if one or both opponents are not humans, one or both of the
   // newly-vacated slots is not filled in.)
   /* 2128:1FD4 */ parray<PlayerReference, 10> recent_human_opponents;
   /* 2240:20EC */ parray<uint8_t, 0x28> unknown_a10;

src/Main.cc

@@ -37,6 +37,7 @@
 #include "StaticGameData.hh"
 #include "Text.hh"
 #include "TextArchive.hh"
+#include "WordSelectSet.hh"
 
 using namespace std;
 
@@ -258,7 +259,7 @@ The actions are:\n\
     Convert a REL or GSL rare table to a JSON rare item set. The resulting JSON\n\
     has the same structure as system/blueburst/rare-table.json.\n\
   generate-dc-serial-number [--domain=DOMAIN] [--subdomain=SUBDOMAIN]\n\
-    Generate a PSO DC serial number. DOMAIN should be 0 for DCv1 or 1 for DCv2;\n\
+    Generate a PSO DC serial number. DOMAIN should be 1 for DCv1 or 2 for DCv2;\n\
     SUBDOMAIN should be 0 for Japanese, 1 for USA, or 2 for Europe.\n\
   generate-all-dc-serial-numbers\n\
     Generate all possible PSO DC serial numbers.\n\

src/ReceiveSubcommands.cc

@@ -434,10 +434,75 @@ static void on_symbol_chat(shared_ptr<Client> c, uint8_t command, uint8_t flag,
   }
 }
 
+template <bool SenderIsBigEndian>
+static void on_word_select_t(shared_ptr<Client> c, uint8_t command, uint8_t, const void* data, size_t size) {
+  const auto& cmd = check_size_t<G_WordSelect_6x74<SenderIsBigEndian>>(data, size);
+  if (c->can_chat && (cmd.client_id == c->lobby_client_id)) {
+    if (command_is_private(command)) {
+      return;
+    }
+
+    auto s = c->require_server_state();
+    auto l = c->require_lobby();
+    if (l->battle_record && l->battle_record->battle_in_progress()) {
+      l->battle_record->add_command(Episode3::BattleRecord::Event::Type::GAME_COMMAND, data, size);
+    }
+
+    unordered_set<shared_ptr<Client>> target_clients;
+    for (const auto& lc : l->clients) {
+      if (lc) {
+        target_clients.emplace(lc);
+      }
+    }
+    for (const auto& watcher_l : l->watcher_lobbies) {
+      for (const auto& lc : watcher_l->clients) {
+        if (lc) {
+          target_clients.emplace(lc);
+        }
+      }
+    }
+    target_clients.erase(c);
+
+    // In non-Ep3 lobbies, Ep3 uses the Ep1&2 word select table.
+    bool is_non_ep3_lobby = (l->episode != Episode::EP3);
+
+    QuestScriptVersion from_version = c->quest_version();
+    if (is_non_ep3_lobby && (from_version == QuestScriptVersion::GC_EP3)) {
+      from_version = QuestScriptVersion::GC_V3;
+    }
+    for (const auto& lc : target_clients) {
+      try {
+        QuestScriptVersion lc_version = lc->quest_version();
+        if (is_non_ep3_lobby && (lc_version == QuestScriptVersion::GC_EP3)) {
+          lc_version = QuestScriptVersion::GC_V3;
+        }
+
+        if (lc->version() == GameVersion::GC) {
+          G_WordSelect_6x74<true> out_cmd = {
+              cmd.subcommand, cmd.size, cmd.client_id.load(),
+              s->word_select_table->translate(cmd.message, from_version, lc_version)};
+          send_command_t(lc, 0x60, 0x00, out_cmd);
+        } else {
+          G_WordSelect_6x74<false> out_cmd = {
+              cmd.subcommand, cmd.size, cmd.client_id.load(),
+              s->word_select_table->translate(cmd.message, from_version, lc_version)};
+          send_command_t(lc, 0x60, 0x00, out_cmd);
+        }
+
+      } catch (const exception& e) {
+        string name = encode_sjis(c->game_data.player()->disp.name);
+        lc->log.warning("Untranslatable Word Select message: %s", e.what());
+        send_text_message_printf(lc, "$C4Untranslatable Word\nSelect message from\n%s", name.c_str());
+      }
+    }
+  }
+}
+
 static void on_word_select(shared_ptr<Client> c, uint8_t command, uint8_t flag, const void* data, size_t size) {
-  const auto& cmd = check_size_t<G_WordSelect_6x74>(data, size);
-  if (c->can_chat && (cmd.header.client_id == c->lobby_client_id)) {
-    forward_subcommand(c, command, flag, data, size);
+  if (c->version() == GameVersion::GC) {
+    on_word_select_t<true>(c, command, flag, data, size);
+  } else {
+    on_word_select_t<false>(c, command, flag, data, size);
   }
 }
 

src/ServerShell.cc

@@ -283,6 +283,8 @@ Proxy session commands:\n\
         this->state->load_level_table();
       } else if (type == "item-tables") {
         this->state->load_item_tables();
+      } else if (type == "word-select") {
+        this->state->load_word_select_table();
       } else if (type == "ep3") {
         this->state->load_ep3_data();
       } else if (type == "quests") {

src/ServerState.cc

@@ -100,6 +100,7 @@ void ServerState::init() {
   this->load_battle_params();
   this->load_level_table();
   this->load_item_tables();
+  this->load_word_select_table();
   this->load_ep3_data();
   this->resolve_ep3_card_names();
   this->load_quest_index();
@@ -912,8 +913,12 @@ void ServerState::load_battle_params() {
 
 void ServerState::load_level_table() {
   config_log.info("Loading level table");
-  this->level_table.reset(new LevelTable(
-      this->load_bb_file("PlyLevelTbl.prs"), true));
+  this->level_table.reset(new LevelTable(this->load_bb_file("PlyLevelTbl.prs"), true));
+}
+
+void ServerState::load_word_select_table() {
+  config_log.info("Loading Word Select table");
+  this->word_select_table.reset(new WordSelectTable(JSON::parse(load_file("system/word-select-table.json"))));
 }
 
 void ServerState::load_item_tables() {

src/ServerState.hh

@@ -21,6 +21,7 @@
 #include "Lobby.hh"
 #include "Menu.hh"
 #include "Quest.hh"
+#include "WordSelectTable.hh"
 
 // Forward declarations due to reference cycles
 class ProxyServer;
@@ -97,6 +98,7 @@ struct ServerState : public std::enable_shared_from_this<ServerState> {
   std::shared_ptr<const TekkerAdjustmentSet> tekker_adjustment_set;
   std::shared_ptr<const ItemParameterTable> item_parameter_table;
   std::shared_ptr<const MagEvolutionTable> mag_evolution_table;
+  std::shared_ptr<const WordSelectTable> word_select_table;
 
   std::shared_ptr<Episode3::TournamentIndex> ep3_tournament_index;
 
@@ -220,6 +222,7 @@ struct ServerState : public std::enable_shared_from_this<ServerState> {
   void load_battle_params();
   void load_level_table();
   void load_item_tables();
+  void load_word_select_table();
   void load_ep3_data();
   void resolve_ep3_card_names();
   void load_quest_index();

src/WordSelectTable.cc

@@ -0,0 +1,117 @@
+#include "WordSelectTable.hh"
+
+#include <inttypes.h>
+
+#include <string>
+#include <vector>
+
+using namespace std;
+
+static void index_add(vector<size_t>& index, uint16_t position, size_t value) {
+  if (position != 0xFFFF) {
+    if (index.size() <= position) {
+      index.resize(position + 1);
+    }
+    index[position] = value;
+  }
+}
+
+WordSelectTable::WordSelectTable(const JSON& json) {
+  this->tokens.reserve(json.size());
+  for (const auto& item : json.as_list()) {
+    JSON dc_value_json = item->at(0);
+    JSON pc_value_json = item->at(1);
+    JSON gc_value_json = item->at(2);
+    JSON ep3_value_json = item->at(3);
+    JSON bb_value_json = item->at(4);
+    uint16_t dc_value = dc_value_json.is_null() ? 0xFFFF : dc_value_json.as_int();
+    uint16_t pc_value = pc_value_json.is_null() ? 0xFFFF : pc_value_json.as_int();
+    uint16_t gc_value = gc_value_json.is_null() ? 0xFFFF : gc_value_json.as_int();
+    uint16_t ep3_value = ep3_value_json.is_null() ? 0xFFFF : ep3_value_json.as_int();
+    uint16_t bb_value = bb_value_json.is_null() ? 0xFFFF : bb_value_json.as_int();
+    this->tokens.emplace_back(Token{
+        .dc_value = dc_value,
+        .pc_value = pc_value,
+        .gc_value = gc_value,
+        .ep3_value = ep3_value,
+        .bb_value = bb_value,
+    });
+    index_add(this->dc_index, dc_value, this->tokens.size() - 1);
+    index_add(this->pc_index, pc_value, this->tokens.size() - 1);
+    index_add(this->gc_index, gc_value, this->tokens.size() - 1);
+    index_add(this->ep3_index, ep3_value, this->tokens.size() - 1);
+    index_add(this->bb_index, bb_value, this->tokens.size() - 1);
+  }
+}
+
+uint16_t WordSelectTable::Token::value_for_version(QuestScriptVersion version) const {
+  switch (version) {
+    case QuestScriptVersion::DC_NTE:
+    case QuestScriptVersion::DC_V1:
+    case QuestScriptVersion::DC_V2:
+      return this->dc_value;
+    case QuestScriptVersion::PC_V2:
+      return this->pc_value;
+    // TODO: Which index does GC_NTE use? Here we presume it's the same as GC,
+    // but this may not be true
+    case QuestScriptVersion::GC_NTE:
+    case QuestScriptVersion::GC_V3:
+    case QuestScriptVersion::XB_V3:
+      return this->gc_value;
+    case QuestScriptVersion::GC_EP3:
+      return this->ep3_value;
+    case QuestScriptVersion::BB_V4:
+      return this->bb_value;
+    default:
+      throw logic_error("invalid word select version");
+  }
+}
+
+WordSelectMessage WordSelectTable::translate(
+    const WordSelectMessage& msg,
+    QuestScriptVersion from_version,
+    QuestScriptVersion to_version) const {
+  const std::vector<size_t>* index;
+  switch (from_version) {
+    case QuestScriptVersion::DC_NTE:
+    case QuestScriptVersion::DC_V1:
+    case QuestScriptVersion::DC_V2:
+      index = &this->dc_index;
+      break;
+    case QuestScriptVersion::PC_V2:
+      index = &this->pc_index;
+      break;
+    // TODO: Which index does GC_NTE use? Here we presume it's the same as GC,
+    // but this may not be true
+    case QuestScriptVersion::GC_NTE:
+    case QuestScriptVersion::GC_V3:
+    case QuestScriptVersion::XB_V3:
+      index = &this->gc_index;
+      break;
+    case QuestScriptVersion::GC_EP3:
+      index = &this->ep3_index;
+      break;
+    case QuestScriptVersion::BB_V4:
+      index = &this->bb_index;
+      break;
+    default:
+      throw logic_error("invalid word select version");
+  }
+
+  WordSelectMessage ret;
+  for (size_t z = 0; z < ret.tokens.size(); z++) {
+    if (msg.tokens[z] == 0xFFFF) {
+      ret.tokens[z] = 0xFFFF;
+    } else {
+      ret.tokens[z] = this->tokens.at(index->at(msg.tokens[z])).value_for_version(to_version);
+      if (ret.tokens[z] == 0xFFFF) {
+        throw runtime_error(string_printf("token %04hX has no translation", msg.tokens[z].load()));
+      }
+    }
+  }
+  ret.num_tokens = msg.num_tokens;
+  ret.target_type = msg.target_type;
+  ret.numeric_parameter = msg.numeric_parameter;
+  ret.unknown_a4 = msg.unknown_a4;
+  return ret;
+}

src/WordSelectTable.hh

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <inttypes.h>
+
+#include <phosg/JSON.hh>
+#include <string>
+#include <vector>
+
+#include "CommandFormats.hh"
+#include "QuestScript.hh"
+
+class WordSelectTable {
+public:
+  explicit WordSelectTable(const JSON& json);
+
+  WordSelectMessage translate(
+      const WordSelectMessage& msg,
+      QuestScriptVersion from_version,
+      QuestScriptVersion to_version) const;
+
+private:
+  struct Token {
+    uint16_t dc_value;
+    uint16_t pc_value;
+    uint16_t gc_value;
+    uint16_t ep3_value;
+    uint16_t bb_value;
+
+    uint16_t value_for_version(QuestScriptVersion version) const;
+  };
+  std::vector<size_t> dc_index;
+  std::vector<size_t> pc_index;
+  std::vector<size_t> gc_index;
+  std::vector<size_t> ep3_index;
+  std::vector<size_t> bb_index;
+  std::vector<Token> tokens;
+};