Split jobs into separate table, achievement points

Author: Simbiat

composer.lock

@@ -105,50 +105,6 @@ public function UpdateOld(int $limit = 1, int $instance = 1): bool|string
         }
     }
 
-    /**
-     * Function to add missing jobs to tracker
-     * @return bool|string
-     */
-    public function UpdateJobs(): bool|string
-    {
-        try {
-            #Cache controller
-            $dbController = HomePage::$dbController;
-            #Get the freshest character ID
-            $characterId = $dbController->selectValue('SELECT `characterid` FROM `ffxiv__character` WHERE `deleted` IS NULL ORDER BY `updated` DESC LIMIT 1;');
-            #Grab its data from Lodestone
-            $character = (new Character((string)$characterId))->getFromLodestone();
-            if (empty($character['jobs'])) {
-                return 'No jobs retrieved for character '.$characterId;
-            }
-            #Sort alphabetically by keys
-            ksort($character['jobs'], SORT_NATURAL);
-            #Prepare string for ALTER
-            $alter = [];
-            #Previous job in the list (for AFTER clause)
-            $previous = '';
-            foreach ($character['jobs'] as $job => $details) {
-                #Remove spaces from the job name
-                $jobNoSpace = preg_replace('/\s*/', '', $job);
-                #Check if job is present as respective column
-                if (!$dbController->checkColumn('ffxiv__character', $jobNoSpace)) {
-                    #Add respective column definition
-                    $alter[] = 'ADD COLUMN `'.$jobNoSpace.'` TINYINT(3) UNSIGNED NOT NULL DEFAULT 0 COMMENT \'Level of '.$job.' job\' AFTER `'.(empty($previous) ? 'pvp_matches' : $previous).'`';
-                }
-                #Update previous column name
-                $previous = $jobNoSpace;
-            }
-            if (empty($alter)) {
-                #Nothing to add
-                return true;
-            }
-            #Generate and run the query
-            return $dbController->query('ALTER TABLE `ffxiv__character` '.implode(', ', $alter).';');
-        } catch (\Throwable $e) {
-            return $e->getMessage()."\r\n".$e->getTraceAsString();
-        }
-    }
-    
     /**
      * Update list of servers
      * @return bool|string

lib/fftracker/Cron.php

@@ -25,6 +25,7 @@ class Character extends Entity
     public array $groups = [];
     public array $jobs = [];
     public array $achievements = [];
+    public int $achievementPoints = 0;
     public array $owned = [];
 
     #Function to get initial data from DB
@@ -45,6 +46,8 @@ protected function getFromDB(): array
         } else {
             $data['username'] = null;
         }
+        #Get jobs
+        $data['jobs'] = HomePage::$dbController->selectAll('SELECT `name`, `level`, `last_change` FROM `ffxiv__character_jobs` LEFT JOIN `ffxiv__jobs` ON `ffxiv__character_jobs`.`jobid`=`ffxiv__jobs`.`jobid` WHERE `ffxiv__character_jobs`.`characterid`=:id ORDER BY `name`;', [':id'=>$this->id]);
         #Get old names. For now returning only the count due to cases of bullying, when the old names are learnt. They are still being collected, though for statistical purposes.
         $data['oldNames'] = HomePage::$dbController->Count('SELECT COUNT(*) FROM `ffxiv__character_names` WHERE `characterid`=:id AND `name`!=:name', [':id'=>$this->id, ':name'=>$data['name']]);
         #Get previous known incarnations (combination of gender and race/clan)
@@ -145,19 +148,8 @@ protected function process(array $fromDB): void
         foreach ($this->achievements as $key=>$achievement) {
             $this->achievements[$key]['time'] = (empty($achievement['time']) ? null : strtotime($achievement['time']));
         }
-        #Remove all already processed elements to converts the rest to jobs array
-        unset(
-            $fromDB['characterid'], $fromDB['userid'], $fromDB['username'], $fromDB['name'], $fromDB['avatar'], $fromDB['biography'], $fromDB['genderid'], $fromDB['datacenter'],
-            $fromDB['registered'], $fromDB['updated'],$fromDB['deleted'], $fromDB['clan'], $fromDB['race'], $fromDB['server'], $fromDB['titleIcon'],
-            $fromDB['region'], $fromDB['city'], $fromDB['cityid'], $fromDB['nameday'], $fromDB['guardian'], $fromDB['guardianid'],
-            $fromDB['servers'], $fromDB['incarnations'], $fromDB['title'], $fromDB['titleid'], $fromDB['dbid'], $fromDB['gcId'],
-            $fromDB['gcrankid'], $fromDB['gc_rank'], $fromDB['gcName'], $fromDB['oldNames'], $fromDB['killedby'],
-            $fromDB['groups'], $fromDB['achievements'],  $fromDB['pvp_matches'], $fromDB['enemyid'],  $fromDB['raceid'],
-        );
-        $this->jobs = $fromDB;
-        foreach ($this->jobs as $key=>$job) {
-            $this->jobs[$key] = (int)$job;
-        }
+        $this->achievementPoints = $fromDB['achievement_points'];
+        $this->jobs = $fromDB['jobs'];
     }
 
     #Function to update the entity
@@ -203,16 +195,23 @@ protected function updateDB(bool $manual = false): string|bool
             if (($this->lodestone['bio'] === '-')) {
                 $this->lodestone['bio'] = null;
             }
+            #Get total achievements points. Using foreach for speed
+            $achievementPoints = 0;
+            if (!empty($this->lodestone['achievements']) && is_array($this->lodestone['achievements'])) {
+                foreach ($this->lodestone['achievements'] as $item) {
+                    $achievementPoints += (int)$item['points'];
+                }
+            }
             #Main query to insert or update a character
             $queries[] = [
                 'INSERT INTO `ffxiv__character`(
-                    `characterid`, `serverid`, `name`, `manual`, `registered`, `updated`, `deleted`, `enemyid`, `biography`, `titleid`, `avatar`, `clanid`, `genderid`, `namedayid`, `guardianid`, `cityid`, `gcrankid`, `pvp_matches`
+                    `characterid`, `serverid`, `name`, `manual`, `registered`, `updated`, `deleted`, `enemyid`, `biography`, `titleid`, `avatar`, `clanid`, `genderid`, `namedayid`, `guardianid`, `cityid`, `gcrankid`, `pvp_matches`, `achievement_points`
                 )
                 VALUES (
-                    :characterid, (SELECT `serverid` FROM `ffxiv__server` WHERE `server`=:server), :name, :manual, UTC_DATE(), CURRENT_TIMESTAMP(), NULL, NULL, :biography, (SELECT `achievementid` as `titleid` FROM `ffxiv__achievement` WHERE `title` IS NOT NULL AND `title`=:title LIMIT 1), :avatar, (SELECT `clanid` FROM `ffxiv__clan` WHERE `clan`=:clan), :genderid, (SELECT `namedayid` FROM `ffxiv__nameday` WHERE `nameday`=:nameday), (SELECT `guardianid` FROM `ffxiv__guardian` WHERE `guardian`=:guardian), (SELECT `cityid` FROM `ffxiv__city` WHERE `city`=:city), `gcrankid` = (SELECT `gcrankid` FROM `ffxiv__grandcompany_rank` WHERE `gc_rank`=:gcRank ORDER BY `gcrankid` LIMIT 1), 0
+                    :characterid, (SELECT `serverid` FROM `ffxiv__server` WHERE `server`=:server), :name, :manual, UTC_DATE(), CURRENT_TIMESTAMP(), NULL, NULL, :biography, (SELECT `achievementid` as `titleid` FROM `ffxiv__achievement` WHERE `title` IS NOT NULL AND `title`=:title LIMIT 1), :avatar, (SELECT `clanid` FROM `ffxiv__clan` WHERE `clan`=:clan), :genderid, (SELECT `namedayid` FROM `ffxiv__nameday` WHERE `nameday`=:nameday), (SELECT `guardianid` FROM `ffxiv__guardian` WHERE `guardian`=:guardian), (SELECT `cityid` FROM `ffxiv__city` WHERE `city`=:city), `gcrankid` = (SELECT `gcrankid` FROM `ffxiv__grandcompany_rank` WHERE `gc_rank`=:gcRank ORDER BY `gcrankid` LIMIT 1), 0, :achievementPoints
                 )
                 ON DUPLICATE KEY UPDATE
-                    `serverid`=(SELECT `serverid` FROM `ffxiv__server` WHERE `server`=:server), `name`=:name, `updated`=CURRENT_TIMESTAMP(), `deleted`=NULL, `enemyid`=NULL, `biography`=:biography, `titleid`=(SELECT `achievementid` as `titleid` FROM `ffxiv__achievement` WHERE `title` IS NOT NULL AND `title`=:title LIMIT 1), `avatar`=:avatar, `clanid`=(SELECT `clanid` FROM `ffxiv__clan` WHERE `clan`=:clan), `genderid`=:genderid, `namedayid`=(SELECT `namedayid` FROM `ffxiv__nameday` WHERE `nameday`=:nameday), `guardianid`=(SELECT `guardianid` FROM `ffxiv__guardian` WHERE `guardian`=:guardian), `cityid`=(SELECT `cityid` FROM `ffxiv__city` WHERE `city`=:city), `gcrankid`=(SELECT `gcrankid` FROM `ffxiv__grandcompany_rank` WHERE `gc_rank` IS NOT NULL AND `gc_rank`=:gcRank ORDER BY `gcrankid` LIMIT 1);',
+                    `serverid`=(SELECT `serverid` FROM `ffxiv__server` WHERE `server`=:server), `name`=:name, `updated`=CURRENT_TIMESTAMP(), `deleted`=NULL, `enemyid`=NULL, `biography`=:biography, `titleid`=(SELECT `achievementid` as `titleid` FROM `ffxiv__achievement` WHERE `title` IS NOT NULL AND `title`=:title LIMIT 1), `avatar`=:avatar, `clanid`=(SELECT `clanid` FROM `ffxiv__clan` WHERE `clan`=:clan), `genderid`=:genderid, `namedayid`=(SELECT `namedayid` FROM `ffxiv__nameday` WHERE `nameday`=:nameday), `guardianid`=(SELECT `guardianid` FROM `ffxiv__guardian` WHERE `guardian`=:guardian), `cityid`=(SELECT `cityid` FROM `ffxiv__city` WHERE `city`=:city), `gcrankid`=(SELECT `gcrankid` FROM `ffxiv__grandcompany_rank` WHERE `gc_rank` IS NOT NULL AND `gc_rank`=:gcRank ORDER BY `gcrankid` LIMIT 1), `achievement_points`=:achievementPoints;',
                 [
                     ':characterid'=>$this->id,
                     ':server'=>$this->lodestone['server'],
@@ -230,24 +229,23 @@ protected function updateDB(bool $manual = false): string|bool
                     ':guardian'=>$this->lodestone['guardian']['name'],
                     ':city'=>$this->lodestone['city']['name'],
                     ':gcRank'=>(empty($this->lodestone['grandCompany']['rank']) ? '' : $this->lodestone['grandCompany']['rank']),
+                    ':achievementPoints' => [$achievementPoints, 'int']
                 ],
             ];
             #Update levels. Doing this in cycle since columns can vary. This can reduce performance, but so far this is the best idea I have to make it as automated as possible
             if (!empty($this->lodestone['jobs'])) {
                 foreach ($this->lodestone['jobs'] as $job=>$level) {
-                    #Remove spaces from the job name
-                    $jobNoSpace = preg_replace('/\s*/', '', $job);
-                    #Check if column exists in order to avoid errors. Checking that level is not empty to not waste time on updating zeros
-                    if (!empty($level['level']) && HomePage::$dbController->checkColumn('ffxiv__character', $jobNoSpace)) {
-                        #Update level
-                        $queries[] = [
-                            'UPDATE `ffxiv__character` SET `'.$jobNoSpace.'`=:level WHERE `characterid`=:characterid;',
-                            [
-                                ':characterid' => $this->id,
-                                ':level' => [(int)$level['level'], 'int'],
-                            ],
-                        ];
-                    }
+                    #Insert job, in case it's missing
+                    $queries[] = ['INSERT IGNORE INTO `ffxiv__jobs` (`name`) VALUES (:job);', [':job' => $job]];
+                    #Update level
+                    $queries[] = [
+                        'INSERT INTO `ffxiv__character_jobs` (`characterid`, `jobid`, `level`, `last_change`) VALUES (:characterid, (SELECT `jobid` FROM `ffxiv__jobs` WHERE `name`=:jobname), :level, CURRENT_TIMESTAMP()) ON DUPLICATE KEY UPDATE `level`=:level, `last_change`=IF(`level`=:level, `last_change`, CURRENT_TIMESTAMP());',
+                        [
+                            ':characterid' => $this->id,
+                            ':jobname' => $job,
+                            ':level' => [(int)$level['level'], 'int'],
+                        ],
+                    ];
                 }
             }
             #Insert server, if it has not been inserted yet. If server is registered at all.
@@ -309,7 +307,7 @@ protected function updateDB(bool $manual = false): string|bool
                         [
                             ':achievementid'=>$achievementid,
                             ':name'=> $item['name'],
-                            ':icon'=> self::removeLodestoneDomain($item['icon']),
+                            ':icon'=> str_replace('.png', '.webp', self::removeLodestoneDomain($item['icon'])),
                             ':points'=> $item['points'],
                         ],
                     ];

lib/fftracker/Entities/Character.php

@@ -34,10 +34,6 @@ protected function generate(array $path): array
         if (empty($outputArray['character']['id'])) {
             return ['http_error' => 404, 'suggested_link' => $this->getLastCrumb()];
         }
-        #Adjust jobs
-        foreach ($outputArray['character']['jobs'] as $job=>$level) {
-            $outputArray['character']['jobs'][$job] = ['name'=>preg_replace('/((?!^)[A-Z])/m', ' $1', $job), 'level'=>$level];
-        }
         #Try to exit early based on modification date
         $this->lastModified($outputArray['character']['dates']['updated']);
         #Continue breadcrumbs

lib/fftracker/Pages/Character.php

@@ -110,45 +110,9 @@ private function getRaw(array &$data, Controller $dbCon): void
     private function getCharacters(array &$data, Controller $dbCon): void
     {
         #Jobs popularity
-        $data['characters']['jobs'] = $dbCon->selectRow(
-            'SELECT
-                                SUM(`Alchemist`) AS `Alchemist`,
-                                SUM(`Armorer`) AS `Armorer`,
-                                SUM(`Astrologian`) AS `Astrologian`,
-                                SUM(`Bard`) AS `Bard`,
-                                SUM(`BlackMage`) AS `Black Mage`,
-                                SUM(`Blacksmith`) AS `Blacksmith`,
-                                SUM(`BlueMage`) AS `Blue Mage`,
-                                SUM(`Botanist`) AS `Botanist`,
-                                SUM(`Carpenter`) AS `Carpenter`,
-                                SUM(`Culinarian`) AS `Culinarian`,
-                                SUM(`Dancer`) AS `Dancer`,
-                                SUM(`DarkKnight`) AS `Dark Knight`,
-                                SUM(`Dragoon`) AS `Dragoon`,
-                                SUM(`Fisher`) AS `Fisher`,
-                                SUM(`Goldsmith`) AS `Goldsmith`,
-                                SUM(`Gunbreaker`) AS `Gunbreaker`,
-                                SUM(`Leatherworker`) AS `Leatherworker`,
-                                SUM(`Machinist`) AS `Machinist`,
-                                SUM(`Miner`) AS `Miner`,
-                                SUM(`Monk`) AS `Monk`,
-                                SUM(`Ninja`) AS `Ninja`,
-                                SUM(`Paladin`) AS `Paladin`,
-                                SUM(`Pictomancer`) AS `Pictomancer`,
-                                SUM(`Reaper`) AS `Reaper`,
-                                SUM(`RedMage`) AS `Red Mage`,
-                                SUM(`Sage`) AS `Sage`,
-                                SUM(`Samurai`) AS `Samurai`,
-                                SUM(`Scholar`) AS `Scholar`,
-                                SUM(`Summoner`) AS `Summoner`,
-                                SUM(`Viper`) AS `Viper`,
-                                SUM(`Warrior`) AS `Warrior`,
-                                SUM(`Weaver`) AS `Weaver`,
-                                SUM(`WhiteMage`) AS `White Mage`
-                            FROM `ffxiv__character`;'
+        $data['characters']['jobs'] = $dbCon->selectPair(
+            'SELECT `name`, SUM(`level`) as `sum` FROM `ffxiv__character_jobs` LEFT JOIN `ffxiv__jobs` ON `ffxiv__jobs`.`jobid`=`ffxiv__character_jobs`.`jobid` GROUP BY `ffxiv__character_jobs`.`jobid` ORDER BY `sum` DESC;'
         );
-        #Sort array
-        arsort($data['characters']['jobs']);
         #Most name changes
         $data['characters']['changes']['name'] = $dbCon->countUnique('ffxiv__character_names', 'characterid', '', 'ffxiv__character', 'INNER', 'characterid', '`tempresult`.`characterid` AS `id`, `ffxiv__character`.`avatar` AS `icon`, \'character\' AS `type`, `ffxiv__character`.`name`', 'DESC', 20, [], true);
         ArrayHelpers::renameColumn($data['characters']['changes']['name'], 'value', 'name');