Add configuration for database connection collation (5.1.x) (#22588)

* Define default collation for TiDB tables (#22271) * Add configuration for database connection collation (#22564) * Backport test changes from #22356
matomo-org · Sep 13, 2024 · 30a811c · 30a811c
1 parent 7f1e128
commit 30a811c
Show file tree

Hide file tree

Showing 28 changed files with 755 additions and 87 deletions.
diff --git a/config/global.ini.php b/config/global.ini.php
@@ -45,6 +45,14 @@
 ; Matomo should work correctly without this setting but we recommend to have a charset set.
 charset = utf8
 
+; In some database setups the collation used for queries and creating tables can have unexpected
+; values, or change after a database version upgrade.
+; If you encounter "Illegal mix of collation" errors, setting this config to the value matching
+; your existing database tables can help.
+; This setting will only be used if "charset" is also set.
+; Matomo should work correctly without this setting but we recommend to have a collation set.
+collation =
+
 ; Database error codes to ignore during updates
 ;
 ;ignore_error_codes[] = 1105
@@ -84,6 +92,7 @@
 type = InnoDB
 schema = Mysql
 charset = utf8mb4
+collation = utf8mb4_general_ci
 enable_ssl = 0
 ssl_ca =
 ssl_cert =

diff --git a/core/Db.php b/core/Db.php
@@ -183,6 +183,7 @@ public static function createReaderDatabaseObject($dbConfig = null)
         $dbConfig['type'] = $masterDbConfig['type'];
         $dbConfig['tables_prefix'] = $masterDbConfig['tables_prefix'];
         $dbConfig['charset'] = $masterDbConfig['charset'];
+        $dbConfig['collation'] = $masterDbConfig['collation'] ?? null;
 
         $db = @Adapter::factory($dbConfig['adapter'], $dbConfig);
 

diff --git a/core/Db/Schema.php b/core/Db/Schema.php
@@ -89,6 +89,27 @@ private function getSchema(): SchemaInterface
         return $this->schema;
     }
 
+    /**
+     * Returns the default collation for a charset.
+     *
+     * @param string $charset
+     * @return string
+     */
+    public function getDefaultCollationForCharset(string $charset): string
+    {
+        return $this->getSchema()->getDefaultCollationForCharset($charset);
+    }
+
+    /**
+     * Get the table options to use for a CREATE TABLE statement.
+     *
+     * @return string
+     */
+    public function getTableCreateOptions(): string
+    {
+        return $this->getSchema()->getTableCreateOptions();
+    }
+
     /**
      * Get the SQL to create a specific Piwik table
      *

diff --git a/core/Db/Schema/Mysql.php b/core/Db/Schema/Mysql.php
diff --git a/core/Db/Schema/Tidb.php b/core/Db/Schema/Tidb.php
@@ -25,8 +25,44 @@ public function supportsComplexColumnUpdates(): bool
         return false;
     }
 
+    public function getDefaultCollationForCharset(string $charset): string
+    {
+        $collation = parent::getDefaultCollationForCharset($charset);
+
+        if ('utf8mb4' === $charset && 'utf8mb4_bin' === $collation) {
+            // replace the TiDB default "utf8mb4_bin" with a better default
+            return 'utf8mb4_0900_ai_ci';
+        }
+
+        return $collation;
+    }
+
     public function getDefaultPort(): int
     {
         return 4000;
     }
+
+    public function getTableCreateOptions(): string
+    {
+        $engine = $this->getTableEngine();
+        $charset = $this->getUsedCharset();
+        $collation = $this->getUsedCollation();
+        $rowFormat = $this->getTableRowFormat();
+
+        if ('utf8mb4' === $charset && '' === $collation) {
+            $collation = 'utf8mb4_0900_ai_ci';
+        }
+
+        $options = "ENGINE=$engine DEFAULT CHARSET=$charset";
+
+        if ('' !== $collation) {
+            $options .= " COLLATE=$collation";
+        }
+
+        if ('' !== $rowFormat) {
+            $options .= " $rowFormat";
+        }
+
+        return $options;
+    }
 }
diff --git a/core/Db/SchemaInterface.php b/core/Db/SchemaInterface.php
@@ -125,10 +125,26 @@ public function addMaxExecutionTimeHintToQuery(string $sql, float $limit): strin
      */
     public function supportsComplexColumnUpdates(): bool;
 
+    /**
+     * Returns the default collation for a charset used by this database engine.
+     *
+     * @param string $charset
+     *
+     * @return string
+     */
+    public function getDefaultCollationForCharset(string $charset): string;
+
     /**
      * Return the default port used by this database engine
      *
      * @return int
      */
     public function getDefaultPort(): int;
+
+    /**
+     * Return the table options to use for a CREATE TABLE statement.
+     *
+     * @return string
+     */
+    public function getTableCreateOptions(): string;
 }
diff --git a/core/Db/Settings.php b/core/Db/Settings.php
@@ -33,6 +33,11 @@ public function getUsedCharset()
         return strtolower($this->getDbSetting('charset'));
     }
 
+    public function getUsedCollation()
+    {
+        return strtolower($this->getDbSetting('collation') ?? '');
+    }
+
     public function getRowFormat()
     {
         return $this->getUsedCharset() === 'utf8mb4' ? 'ROW_FORMAT=DYNAMIC' : '';

diff --git a/core/DbHelper.php b/core/DbHelper.php
@@ -210,7 +210,7 @@ public static function tableHasIndex($table, $indexName)
      * @return string
      * @throws Tracker\Db\DbException
      */
-    public static function getDefaultCharset()
+    public static function getDefaultCharset(): string
     {
         $result = Db::get()->fetchRow("SHOW CHARACTER SET LIKE 'utf8mb4'");
 
@@ -233,6 +233,19 @@ public static function getDefaultCharset()
         return 'utf8mb4';
     }
 
+    /**
+     * Returns the default collation for a charset.
+     *
+     * @param string $charset
+     *
+     * @return string
+     * @throws Exception
+     */
+    public static function getDefaultCollationForCharset(string $charset): string
+    {
+        return Schema::getInstance()->getDefaultCollationForCharset($charset);
+    }
+
     /**
      * Returns sql queries to convert all installed tables to utf8mb4
      *

diff --git a/core/Tracker/Db/Mysqli.php b/core/Tracker/Db/Mysqli.php
@@ -26,6 +26,7 @@ class Mysqli extends Db
     protected $username;
     protected $password;
     protected $charset;
+    protected $collation;
     protected $activeTransaction = false;
 
     protected $enable_ssl;
@@ -57,11 +58,12 @@ public function __construct($dbInfo, $driverName = 'mysql')
             $this->port = (int)$dbInfo['port'];
             $this->socket = null;
         }
+
         $this->dbname = $dbInfo['dbname'];
         $this->username = $dbInfo['username'];
         $this->password = $dbInfo['password'];
-        $this->charset = isset($dbInfo['charset']) ? $dbInfo['charset'] : null;
-
+        $this->charset = $dbInfo['charset'] ?? null;
+        $this->collation = $dbInfo['collation'] ?? null;
 
         if (!empty($dbInfo['enable_ssl'])) {
             $this->enable_ssl = $dbInfo['enable_ssl'];
@@ -133,8 +135,17 @@ public function connect()
             throw new DbException("Connect failed: " . mysqli_connect_error());
         }
 
-        if ($this->charset && !mysqli_set_charset($this->connection, $this->charset)) {
-            throw new DbException("Set Charset failed: " . mysqli_error($this->connection));
+        if ($this->charset && $this->collation) {
+            // mysqli_set_charset does not support setting a collation
+            $query = "SET NAMES '" . $this->charset . "' COLLATE '" . $this->collation . "'";
+
+            if (!mysqli_query($this->connection, $query)) {
+                throw new DbException("Set charset/connection collation failed: " . mysqli_error($this->connection));
+            }
+        } elseif ($this->charset) {
+            if (!mysqli_set_charset($this->connection, $this->charset)) {
+                throw new DbException("Set Charset failed: " . mysqli_error($this->connection));
+            }
         }
 
         $this->password = '';

diff --git a/core/Tracker/Db/Pdo/Mysql.php b/core/Tracker/Db/Pdo/Mysql.php
@@ -26,13 +26,33 @@ class Mysql extends Db
      * @var PDO
      */
     protected $connection = null;
+
+    /**
+     * @var string
+     */
     protected $dsn;
+
+    /**
+     * @var string
+     */
     private $username;
+
+    /**
+     * @var string
+     */
     private $password;
+
+    /**
+     * @var string|null
+     */
     protected $charset;
 
-    protected $mysqlOptions = array();
+    /**
+     * @var string|null
+     */
+    private $collation;
 
+    protected $mysqlOptions = [];
 
     protected $activeTransaction = false;
 
@@ -58,8 +78,11 @@ public function __construct($dbInfo, $driverName = 'mysql')
         if (isset($dbInfo['charset'])) {
             $this->charset = $dbInfo['charset'];
             $this->dsn .= ';charset=' . $this->charset;
-        }
 
+            if (!empty($dbInfo['collation'])) {
+                $this->collation = $dbInfo['collation'];
+            }
+        }
 
         if (isset($dbInfo['enable_ssl']) && $dbInfo['enable_ssl']) {
             if (!empty($dbInfo['ssl_key'])) {
@@ -409,6 +432,11 @@ private function establishConnection(): void
          */
         if (!empty($this->charset)) {
             $sql = "SET NAMES '" . $this->charset . "'";
+
+            if (!empty($this->collation)) {
+                $sql .= " COLLATE '" . $this->collation . "'";
+            }
+
             $this->connection->exec($sql);
         }
     }

diff --git a/core/Updater/Migration/Db/CreateTable.php b/core/Updater/Migration/Db/CreateTable.php
@@ -9,7 +9,7 @@
 
 namespace Piwik\Updater\Migration\Db;
 
-use Piwik\Db;
+use Piwik\Db\Schema;
 
 /**
  * @see Factory::createTable()
@@ -19,12 +19,11 @@ class CreateTable extends Sql
 {
     /**
      * Constructor.
-     * @param Db\Settings $dbSettings
      * @param string $table Prefixed table name
      * @param string|string[] $columnNames array(columnName => columnValue)
      * @param string|string[] $primaryKey one or multiple columns that define the primary key
      */
-    public function __construct(Db\Settings $dbSettings, $table, $columnNames, $primaryKey)
+    public function __construct($table, $columnNames, $primaryKey)
     {
         $columns = array();
         foreach ($columnNames as $column => $type) {
@@ -35,15 +34,12 @@ public function __construct(Db\Settings $dbSettings, $table, $columnNames, $prim
             $columns[] = sprintf('PRIMARY KEY ( `%s` )', implode('`, `', $primaryKey));
         }
 
-
-        $sql = rtrim(sprintf(
-            'CREATE TABLE `%s` (%s) ENGINE=%s DEFAULT CHARSET=%s %s',
+        $sql = sprintf(
+            'CREATE TABLE `%s` (%s) %s',
             $table,
             implode(', ', $columns),
-            $dbSettings->getEngine(),
-            $dbSettings->getUsedCharset(),
-            $dbSettings->getRowFormat()
-        ));
+            Schema::getInstance()->getTableCreateOptions()
+        );
 
         parent::__construct($sql, static::ERROR_CODE_TABLE_EXISTS);
     }