Merge pull request #5823 from WoltLab/spider-list

Remove the storage of spiders in the database
WoltLab · Feb 27, 2024 · 4c0cfbe · 4c0cfbe
2 parents 295ce35 + 198f05f
commit 4c0cfbe
Show file tree

Hide file tree

Showing 25 changed files with 883 additions and 351 deletions.
diff --git a/XSD/spiderList.xsd b/XSD/spiderList.xsd
diff --git a/com.woltlab.wcf/cronjob.xml b/com.woltlab.wcf/cronjob.xml
@@ -8,12 +8,6 @@
 			<expression type="random">@daily</expression>
 			<canbedisabled>0</canbedisabled>
 		</cronjob>
-		<cronjob name="com.woltlab.wcf.refreshSearchRobots">
-			<classname>wcf\system\cronjob\RefreshSearchRobotsCronjob</classname>
-			<description>Refreshes list of search robots</description>
-			<description language="de">Aktualisiert die Liste der Suchroboter</description>
-			<expression type="random">@monthly</expression>
-		</cronjob>
 		<cronjob name="com.woltlab.wcf.dailyCleanUp">
 			<classname>wcf\system\cronjob\DailyCleanUpCronjob</classname>
 			<description>Executes daily Cleanup</description>
@@ -142,4 +136,9 @@
 			<canbedisabled>0</canbedisabled>
 		</cronjob>
 	</import>
+	<delete>
+		<cronjob name="com.woltlab.wcf.refreshSearchRobots">
+			<classname>wcf\system\cronjob\RefreshSearchRobotsCronjob</classname>
+		</cronjob>
+	</delete>
 </data>
diff --git a/com.woltlab.wcf/templates/usersOnlineList.tpl b/com.woltlab.wcf/templates/usersOnlineList.tpl
@@ -91,15 +91,16 @@
 			<dl class="plain inlineDataList small">
 				<dt>{lang}wcf.user.usersOnline.ipAddress{/lang}</dt>
 				<dd title="{$user->getFormattedIPAddress()}">{@$user->getFormattedIPAddress()|ipSearch}</dd>
-				
-				{if !$user->spiderID}
+
+				{if !$user->spiderIdentifier}
 					<dt>{lang}wcf.user.usersOnline.userAgent{/lang}</dt>
 					<dd title="{$user->userAgent}">{$user->getBrowser()|truncate:30}</dd>
 				{/if}
 			</dl>
 		{/if}
 	{/capture}
-
+
+	{assign var=spider value=$user->getSpider()}
 	{if $user->userID}
 		{* member *}
 		{capture append=usersOnlineList}
@@ -131,7 +132,7 @@
 		{/capture}
 
 		{assign var=usersOnline value=$usersOnline+1}
-	{elseif $user->spiderID}
+	{elseif $spider !== null}
 		{* search robot *}
 		{capture append=robotsOnlineList}
 			<li>
@@ -140,7 +141,9 @@
 
 					<div class="details userInformation">
 						<div class="containerHeadline">
-							<h3>{if $user->getSpider()->spiderURL}<a {anchorAttributes url=$user->getSpider()->spiderURL}>{$user->getSpider()->spiderName}</a>{else}{$user->getSpider()->spiderName}{/if}</h3>
+							<h3>{if $spider->url}
+									<a {anchorAttributes url=$spider->url}>{$spider->name}</a>{else}{$spider->name}{/if}
+							</h3>
 							{@$locationData}
 						</div>
 

diff --git a/wcfsetup/install/files/acp/database/update_com.woltlab.wcf_6.1_spider_step1.php b/wcfsetup/install/files/acp/database/update_com.woltlab.wcf_6.1_spider_step1.php
@@ -0,0 +1,37 @@
+<?php
+
+/**
+ * Drops old spider related columns and index from `wcf1_session` table.
+ *
+ * @author      Olaf Braun
+ * @copyright   2001-2024 WoltLab GmbH
+ * @license     GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
+ * @since       6.1
+ */
+
+use wcf\system\database\table\column\IntDatabaseTableColumn;
+use wcf\system\database\table\index\DatabaseTableForeignKey;
+use wcf\system\database\table\index\DatabaseTableIndex;
+use wcf\system\database\table\PartialDatabaseTable;
+
+return [
+    PartialDatabaseTable::create('wcf1_session')
+        ->columns([
+            IntDatabaseTableColumn::create('spiderID')
+                ->length(10)
+                ->drop(),
+        ])
+        ->indices([
+            DatabaseTableIndex::create('packageID')
+                ->columns(['lastActivityTime', 'spiderID'])
+                ->drop(),
+        ])
+        ->foreignKeys([
+            DatabaseTableForeignKey::create()
+                ->columns(['spiderID'])
+                ->referencedTable('wcf1_spider')
+                ->referencedColumns(['spiderID'])
+                ->onDelete('CASCADE')
+                ->drop(),
+        ]),
+];
diff --git a/wcfsetup/install/files/acp/database/update_com.woltlab.wcf_6.1_spider_step2.php b/wcfsetup/install/files/acp/database/update_com.woltlab.wcf_6.1_spider_step2.php
@@ -0,0 +1,29 @@
+<?php
+
+/**
+ * Drops olds `wcf1_spider` table and create new columns to identify spiders in `wcf1_session` table.
+ *
+ * @author      Olaf Braun
+ * @copyright   2001-2024 WoltLab GmbH
+ * @license     GNU Lesser General Public License <http://opensource.org/licenses/lgpl-license.php>
+ * @since       6.1
+ */
+
+use wcf\system\database\table\column\VarcharDatabaseTableColumn;
+use wcf\system\database\table\index\DatabaseTableIndex;
+use wcf\system\database\table\PartialDatabaseTable;
+
+return [
+    PartialDatabaseTable::create('wcf1_spider')
+        ->drop(),
+    PartialDatabaseTable::create('wcf1_session')
+        ->columns([
+            VarcharDatabaseTableColumn::create('spiderIdentifier')
+                ->length(191)
+                ->defaultValue(null),
+        ])
+        ->indices([
+            DatabaseTableIndex::create('packageID')
+                ->columns(['lastActivityTime', 'spiderIdentifier']),
+        ]),
+];
diff --git a/wcfsetup/install/files/lib/action/AbstractOauth2Action.class.php b/wcfsetup/install/files/lib/action/AbstractOauth2Action.class.php
@@ -41,7 +41,7 @@ public function readParameters()
     {
         parent::readParameters();
 
-        if (WCF::getSession()->spiderID) {
+        if (WCF::getSession()->spiderIdentifier) {
             throw new PermissionDeniedException();
         }
     }

diff --git a/wcfsetup/install/files/lib/action/TwitterAuthAction.class.php b/wcfsetup/install/files/lib/action/TwitterAuthAction.class.php
@@ -47,7 +47,7 @@ public function readParameters()
     {
         parent::readParameters();
 
-        if (WCF::getSession()->spiderID) {
+        if (WCF::getSession()->spiderIdentifier) {
             throw new PermissionDeniedException();
         }
     }

diff --git a/wcfsetup/install/files/lib/data/session/Session.class.php b/wcfsetup/install/files/lib/data/session/Session.class.php
@@ -15,7 +15,7 @@
  * @property-read   int|null $pageObjectID       id of the object the latest page visited belongs to
  * @property-read   int|null $parentPageID       id of the parent page of latest page visited
  * @property-read   int|null $parentPageObjectID id of the object the parent page of latest page visited belongs to
- * @property-read   int $spiderID       id of the spider the session belongs to
+ * @property-read   ?string $spiderIdentifier identifier of the spider
  */
 class Session extends ACPSession
 {

diff --git a/wcfsetup/install/files/lib/data/spider/Spider.class.php b/wcfsetup/install/files/lib/data/spider/Spider.class.php
diff --git a/wcfsetup/install/files/lib/data/spider/SpiderAction.class.php b/wcfsetup/install/files/lib/data/spider/SpiderAction.class.php
diff --git a/wcfsetup/install/files/lib/data/spider/SpiderEditor.class.php b/wcfsetup/install/files/lib/data/spider/SpiderEditor.class.php
diff --git a/wcfsetup/install/files/lib/data/spider/SpiderList.class.php b/wcfsetup/install/files/lib/data/spider/SpiderList.class.php
diff --git a/wcfsetup/install/files/lib/data/user/online/UserOnline.class.php b/wcfsetup/install/files/lib/data/user/online/UserOnline.class.php
@@ -3,11 +3,11 @@
 namespace wcf\data\user\online;
 
 use wcf\data\page\PageCache;
-use wcf\data\spider\Spider;
 use wcf\data\user\UserProfile;
-use wcf\system\cache\builder\SpiderCacheBuilder;
 use wcf\system\event\EventHandler;
 use wcf\system\page\handler\IOnlineLocationPageHandler;
+use wcf\system\spider\Spider;
+use wcf\system\spider\SpiderHandler;
 use wcf\system\WCF;
 use wcf\util\StringUtil;
 use wcf\util\UserAgent;
@@ -24,6 +24,7 @@
  * @property-read   int|null $pageObjectID       id of the object the last visited page belongs to
  * @property-read   int|null $parentPageObjectID id of the parent of the object the last visited page belongs to
  * @property-read   string|null $userOnlineMarking  HTML code used to print the formatted name of a user group member
+ * @property-read   ?string $spiderIdentifier identifier of the spider
  */
 class UserOnline extends UserProfile
 {
@@ -35,9 +36,8 @@ class UserOnline extends UserProfile
 
     /**
      * spider object
-     * @var Spider
      */
-    protected $spider;
+    protected ?Spider $spider;
 
     /**
      * Returns the formatted username.
@@ -152,18 +152,15 @@ public function getBrowser()
 
     /**
      * Returns the spider object
-     *
-     * @return  Spider|null
      */
-    public function getSpider()
+    public function getSpider(): ?Spider
     {
-        if (!$this->spiderID) {
+        if (!$this->spiderIdentifier) {
             return null;
         }
 
-        if ($this->spider === null) {
-            $spiderList = SpiderCacheBuilder::getInstance()->getData();
-            $this->spider = $spiderList[$this->spiderID];
+        if (!isset($this->spider)) {
+            $this->spider = SpiderHandler::getInstance()->getSpider($this->spiderIdentifier);
         }
 
         return $this->spider;

diff --git a/wcfsetup/install/files/lib/data/user/online/UsersOnlineList.class.php b/wcfsetup/install/files/lib/data/user/online/UsersOnlineList.class.php
@@ -100,7 +100,7 @@ public function readObjects()
     public function readStats()
     {
         $conditionBuilder = clone $this->getConditionBuilder();
-        $conditionBuilder->add('session.spiderID IS NULL');
+        $conditionBuilder->add('session.spiderIdentifier IS NULL');
 
         $sql = "SELECT      user_option_value.userOption" . User::getUserOptionID('canViewOnlineStatus') . " AS canViewOnlineStatus, session.userID
                 FROM        wcf" . WCF_N . "_session session

diff --git a/wcfsetup/install/files/lib/page/UserPage.class.php b/wcfsetup/install/files/lib/page/UserPage.class.php
@@ -198,7 +198,7 @@ public function assignVariables()
     public function show()
     {
         // update profile hits
-        if ($this->user->userID != WCF::getUser()->userID && !WCF::getSession()->spiderID && !$this->user->isProtected()) {
+        if ($this->user->userID != WCF::getUser()->userID && !WCF::getSession()->spiderIdentifier && !$this->user->isProtected()) {
             $editor = new UserEditor($this->user->getDecoratedObject());
             $editor->updateCounters(['profileHits' => 1]);
 

diff --git a/wcfsetup/install/files/lib/page/UsersOnlineListPage.class.php b/wcfsetup/install/files/lib/page/UsersOnlineListPage.class.php
@@ -91,18 +91,18 @@ protected function initObjectList()
         $this->objectList->checkRecord();
 
         if (!USERS_ONLINE_SHOW_ROBOTS) {
-            $this->objectList->getConditionBuilder()->add('session.spiderID IS NULL');
+            $this->objectList->getConditionBuilder()->add('session.spiderIdentifier IS NULL');
         }
         if (!USERS_ONLINE_SHOW_GUESTS) {
             if (USERS_ONLINE_SHOW_ROBOTS) {
-                $this->objectList->getConditionBuilder()->add('(session.userID IS NOT NULL OR session.spiderID IS NOT NULL)');
+                $this->objectList->getConditionBuilder()->add('(session.userID IS NOT NULL OR session.spiderIdentifier IS NOT NULL)');
             } else {
                 $this->objectList->getConditionBuilder()->add('session.userID IS NOT NULL');
             }
         }
 
         $this->objectList->sqlSelects .= ", CASE WHEN session.userID IS NULL THEN 1 ELSE 0 END AS userIsGuest";
-        $this->objectList->sqlSelects .= ", CASE WHEN session.spiderID IS NOT NULL THEN 1 ELSE 0 END AS userIsRobot";
+        $this->objectList->sqlSelects .= ", CASE WHEN session.spiderIdentifier IS NOT NULL THEN 1 ELSE 0 END AS userIsRobot";
     }
 
     /**