backport(master): fix(site-backup): Repair a crashed global search table on the site's bench (#7667)

press/agent.py

@@ -408,6 +408,16 @@ def deactivate_site(self, site, reference_doctype=None, reference_name=None):
 			reference_name=reference_name,
 		)
 
+	def fix_global_search(self, site, reference_doctype=None, reference_name=None):
+		return self.create_agent_job(
+			"Fix global search",
+			f"benches/{site.bench}/sites/{site.name}/fix_global_search",
+			bench=site.bench,
+			site=site.name,
+			reference_doctype=reference_doctype,
+			reference_name=reference_name,
+		)
+
 	def update_site(
 		self,
 		site,

press/fixtures/agent_job_type.json

@@ -3262,5 +3262,29 @@
 				"step_name": "Upload Audit Logs To S3"
 			}
 		]
+	},
+	{
+		"disabled_auto_retry": 0,
+		"docstatus": 0,
+		"doctype": "Agent Job Type",
+		"max_retry_count": 3,
+		"modified": "2026-10-05 12:00:00.000000",
+		"name": "Fix global search",
+		"request_method": "POST",
+		"request_path": "/benches/{bench}/sites/{site}/fix_global_search",
+		"steps": [
+			{
+				"parent": "Fix global search",
+				"parentfield": "steps",
+				"parenttype": "Agent Job Type",
+				"step_name": "Truncate Global Search Table"
+			},
+			{
+				"parent": "Fix global search",
+				"parentfield": "steps",
+				"parenttype": "Agent Job Type",
+				"step_name": "Rebuild global search"
+			}
+		]
 	}
 ]

press/press/doctype/site_backup/site_backup.py

@@ -426,21 +426,22 @@ def autocorrect_bench_permissions(self):
 		return False
 
 	def fix_global_search_indexes(self):
-		"""
-		Run this whenever Backup Job fails because of broken global search indexes and regenerate them.
-		"""
-		job = frappe.db.get_value("Agent Job", self.job, ["bench", "server", "output"], as_dict=True)
+		"""Rebuild __global_search when the backup failed because that table is crashed."""
+		job = frappe.db.get_value("Agent Job", self.job, ["output", "traceback"], as_dict=True)
+		if not is_global_search_crashed(f"{job.output or ''}\n{job.traceback or ''}"):
+			return
 
-		if job.output and "Couldn't execute 'show create table `__global_search`'" in job.output:
-			try:
-				agent = Agent(self.server)
-				agent.create_agent_job("Fix global search", "fix_global_search")
-			except Exception:
-				frappe.log_error(
-					"Failed to fix global search indexes",
-					reference_doctype=self.doctype,
-					reference_name=self.name,
-				)
+		try:
+			site = frappe.get_doc("Site", self.site)
+			Agent(site.server).fix_global_search(
+				site, reference_doctype=self.doctype, reference_name=self.name
+			)
+		except Exception:
+			frappe.log_error(
+				"Failed to fix global search indexes",
+				reference_doctype=self.doctype,
+				reference_name=self.name,
+			)
 
 	@classmethod
 	def offsite_backup_exists(cls, site: str, day: date) -> bool:
@@ -506,6 +507,13 @@ def track_offsite_backups(site: str, backup_data: dict, offsite_backup_data: dic
 	)
 
 
+def is_global_search_crashed(output: str) -> bool:
+	# MyISAM marks __global_search crashed after an unclean MariaDB stop, and mariadb-dump stops on it
+	return "__global_search" in output and (
+		"marked as crashed" in output or "Couldn't execute 'show create table `__global_search`'" in output
+	)
+
+
 def process_backup_site_job_update(job):
 	backups = frappe.get_all("Site Backup", fields=["name", "status"], filters={"job": job.name}, limit=1)
 	if not backups:

press/press/doctype/site_backup/test_site_backup.py

@@ -13,10 +13,16 @@
 from press.press.doctype.remote_file.test_remote_file import create_test_remote_file
 from press.press.doctype.site.test_site import create_test_site
 from press.press.doctype.site_activity.test_site_activity import create_test_site_activity
+from press.press.doctype.site_backup.site_backup import is_global_search_crashed
 
 if TYPE_CHECKING:
 	from datetime import datetime
 
+CRASHED_GLOBAL_SEARCH_OUTPUT = (
+	"mariadb-dump: Couldn't execute 'show create table `__global_search`': Table"
+	" './_db/__global_search' is marked as crashed and last (automatic?) repair failed (144)"
+)
+
 
 @patch.object(AgentJob, "enqueue_http_request", new=Mock())
 def create_test_site_backup(
@@ -236,6 +242,45 @@ def test_backup_job_callback_with_offsite(self):
 		self.assertTrue(self.site_backup.remote_private_file)
 		self.assertTrue(self.site_backup.remote_config_file)
 
+	def _fail_backup_job(self, output: str):
+		self.job.db_set("status", "Failure")
+		self.job.db_set("output", output)
+		process_job_updates(self.job.name)
+
+	def _fix_global_search_jobs(self):
+		return frappe.get_all(
+			"Agent Job",
+			filters={"job_type": "Fix global search", "site": self.site.name},
+			fields=["request_path", "bench", "server", "reference_doctype", "reference_name"],
+		)
+
+	@patch.object(AgentJob, "enqueue_http_request", new=Mock())
+	def test_backup_failing_on_crashed_global_search_starts_fix_job_on_site_bench(self):
+		self._fail_backup_job(CRASHED_GLOBAL_SEARCH_OUTPUT)
+
+		jobs = self._fix_global_search_jobs()
+		self.assertEqual(len(jobs), 1)
+		self.assertEqual(
+			jobs[0].request_path, f"benches/{self.site.bench}/sites/{self.site.name}/fix_global_search"
+		)
+		self.assertEqual(jobs[0].bench, self.site.bench)
+		self.assertEqual(jobs[0].server, self.site.server)
+		self.assertEqual(jobs[0].reference_name, self.site_backup.name)
+
+	@patch.object(AgentJob, "enqueue_http_request", new=Mock())
+	def test_backup_failing_for_other_reason_does_not_start_fix_global_search_job(self):
+		self._fail_backup_job("mariadb-dump: Got error: 2013: Lost connection to server during query")
+
+		self.assertEqual(self._fix_global_search_jobs(), [])
+
+	def test_crashed_global_search_is_detected_when_dump_fails_on_a_select(self):
+		output = (
+			"mariadb-dump: Error 145: Table './_db/__global_search' is marked as crashed"
+			" and should be repaired when dumping table `__global_search` at row: 0"
+		)
+		self.assertTrue(is_global_search_crashed(output))
+		self.assertFalse(is_global_search_crashed("Table './_db/tabUser' is marked as crashed"))
+
 	def test_archiving_site_with_offsite_backup_creates_site_backup_record(self):
 		"""
 		When a site is archived after taking an offsite backup, test if a Site Backup record is created from the 'Archive Site' agent job's response.