diff --git a/administrator/components/com_admin/models/sysinfo.php b/administrator/components/com_admin/models/sysinfo.php index 324e352b6872a..be20359889f2c 100644 --- a/administrator/components/com_admin/models/sysinfo.php +++ b/administrator/components/com_admin/models/sysinfo.php @@ -141,6 +141,7 @@ public function &getInfo() $this->info['php'] = php_uname(); $this->info['dbversion'] = $db->getVersion(); $this->info['dbcollation'] = $db->getCollation(); + $this->info['dbconnectioncollation'] = $db->getConnectionCollation(); $this->info['phpversion'] = phpversion(); $this->info['server'] = isset($_SERVER['SERVER_SOFTWARE']) ? $_SERVER['SERVER_SOFTWARE'] : getenv('SERVER_SOFTWARE'); $this->info['sapi_name'] = php_sapi_name(); diff --git a/administrator/components/com_admin/sql/updates/mysql/3.5.0-2015-07-01.sql b/administrator/components/com_admin/sql/updates/mysql/3.5.0-2015-07-01.sql new file mode 100644 index 0000000000000..7ee66ff2de6c4 --- /dev/null +++ b/administrator/components/com_admin/sql/updates/mysql/3.5.0-2015-07-01.sql @@ -0,0 +1,89 @@ +-- WARNING: Do not rename this file with a different date. It MUST run before any other table updates when upgrading to Joomla! 3.5.0 + +-- Index and field changes to cater for UTF-8 Multibyte (utf8mb4) +ALTER TABLE `#__menu` DROP KEY `idx_client_id_parent_id_alias_language`, ADD UNIQUE KEY `idx_client_id_parent_id_alias_language` (`client_id`,`parent_id`,`alias`(191),`language`); + +ALTER TABLE `#__redirect_links` DROP KEY `idx_link_old`, ADD UNIQUE KEY `idx_link_old` (`old_url`(191)); + +ALTER TABLE `#__session` MODIFY `session_id` varchar(191) NOT NULL DEFAULT ''; + +ALTER TABLE `#__user_keys` MODIFY `series` varchar(191) NOT NULL; + +-- Convert utf8_bin collated columns to utf8mb4_bin collation +ALTER TABLE `#__banners` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT ''; +ALTER TABLE `#__categories` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT ''; +ALTER TABLE `#__contact_details` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT ''; +ALTER TABLE `#__content` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT ''; +ALTER TABLE `#__menu` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'The SEF alias of the menu item.'; +ALTER TABLE `#__newsfeeds` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT ''; +ALTER TABLE `#__tags` MODIFY `alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT ''; +ALTER TABLE `#__ucm_content` MODIFY `core_alias` varchar(191) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT ''; + +-- Convert all tables to UTF-8 Multibyte (utf8mb4) +ALTER TABLE `#__assets` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__associations` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__banners` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__banner_clients` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__banner_tracks` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__categories` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__contact_details` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__content` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__content_frontpage` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__content_rating` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__content_types` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__contentitem_tag_map` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__core_log_searches` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__extensions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_filters` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_terms0` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_terms1` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_terms2` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_terms3` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_terms4` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_terms5` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_terms6` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_terms7` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_terms8` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_terms9` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_termsa` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_termsb` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_termsc` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_termsd` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_termse` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_links_termsf` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_taxonomy` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_taxonomy_map` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_terms` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_terms_common` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_tokens` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_tokens_aggregate` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__finder_types` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__languages` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__menu` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__menu_types` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__messages` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__messages_cfg` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__modules` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__modules_menu` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__newsfeeds` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__overrider` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__postinstall_messages` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__redirect_links` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__schemas` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__session` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__tags` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__template_styles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__ucm_base` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__ucm_content` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__ucm_history` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__updates` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__update_sites` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__update_sites_extensions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__usergroups` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__users` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__user_keys` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__user_notes` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__user_profiles` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__user_usergroup_map` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; +ALTER TABLE `#__viewlevels` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; \ No newline at end of file diff --git a/administrator/components/com_admin/views/sysinfo/tmpl/default_system.php b/administrator/components/com_admin/views/sysinfo/tmpl/default_system.php index 539470bfcc506..f9f49d75cadcd 100644 --- a/administrator/components/com_admin/views/sysinfo/tmpl/default_system.php +++ b/administrator/components/com_admin/views/sysinfo/tmpl/default_system.php @@ -52,6 +52,14 @@ info['dbcollation']; ?> + + + + + + info['dbconnectioncollation']; ?> + + diff --git a/administrator/components/com_installer/models/database.php b/administrator/components/com_installer/models/database.php index 75165013a7b41..e6a4398f82a52 100644 --- a/administrator/components/com_installer/models/database.php +++ b/administrator/components/com_installer/models/database.php @@ -63,6 +63,9 @@ public function fix() $installer = new JoomlaInstallerScript; $installer->deleteUnexistingFiles(); $this->fixDefaultTextFilters(); + + // Finally, make sure the database is converted to utf8mb4 if supported by the server + $this->convertTablesToUtf8mb4(); } /** @@ -255,4 +258,45 @@ public function fixDefaultTextFilters() } } } + + public function convertTablesToUtf8mb4() + { + $db = JFactory::getDbo(); + + // If the database does not have UTF-8 Multibyte (utf8mb4) support we can't do much about it. + if (!$db->hasUTF8mb4Support()) + { + return; + } + + // Get the SQL file to convert the core tables. Yes, this is hardcoded because we have all sorts of index + // conversions and funky things we can't automate in core tables without an actual SQL file. + $serverType = $db->getServerType(); + $fileName = JPATH_ADMINISTRATOR . "/components/com_admin/sql/updates/$serverType/3.5.0-2015-01-01.sql"; + + if (!is_file($fileName)) + { + return; + } + + $fileContents = @file_get_contents($fileName); + $queries = $db->splitSql($fileContents); + + if (empty($queries)) + { + return; + } + + foreach ($queries as $query) + { + try + { + $db->setQuery($query)->execute(); + } + catch (Exception $e) + { + // If the query fails we will go on. It probably means we've already done this conversion. + } + } + } } diff --git a/administrator/language/en-GB/en-GB.com_admin.ini b/administrator/language/en-GB/en-GB.com_admin.ini index d5f1082afa65e..8255ed4ef8309 100644 --- a/administrator/language/en-GB/en-GB.com_admin.ini +++ b/administrator/language/en-GB/en-GB.com_admin.ini @@ -9,6 +9,7 @@ COM_ADMIN_CACHE_DIRECTORY="(Cache Directory)" COM_ADMIN_CLEAR_RESULTS="Clear results" COM_ADMIN_CONFIGURATION_FILE="Configuration File" COM_ADMIN_DATABASE_COLLATION="Database Collation" +COM_ADMIN_DATABASE_CONNECTION_COLLATION="Database Connection Collation" COM_ADMIN_DATABASE_VERSION="Database Version" COM_ADMIN_DIRECTORY="Directory" COM_ADMIN_DIRECTORY_PERMISSIONS="Directory Permissions" diff --git a/installation/model/database.php b/installation/model/database.php index d84303c05e92c..dba39a59ba213 100644 --- a/installation/model/database.php +++ b/installation/model/database.php @@ -896,6 +896,23 @@ public function populateDatabase($db, $schema) // If the query isn't empty and is not a MySQL or PostgreSQL comment, execute it. if (!empty($query) && ($query{0} != '#') && ($query{0} != '-')) { + /** + * If we don't have UTF-8 Multibyte support we'll have to convert queries to plain UTF-8 + * + * Note: the JDatabaseDriver::convertUtf8mb4QueryToUtf8 performs the conversion ONLY when + * necessary, so there's no need to check the conditions in JInstaller. + */ + $query = $db->convertUtf8mb4QueryToUtf8($query); + + /** + * This is a query which was supposed to convert tables to utf8mb4 charset but the server doesn't + * support utf8mb4. Therefore we don't have to run it, it has no effect and it's a mere waste of time. + */ + if (!$db->hasUTF8mb4Support() && stristr($query, 'CONVERT TO CHARACTER SET utf8 ')) + { + continue; + } + // Execute the query. $db->setQuery($query); diff --git a/installation/sql/mysql/joomla.sql b/installation/sql/mysql/joomla.sql index 652ff67d6410d..d0405a82fcea6 100644 --- a/installation/sql/mysql/joomla.sql +++ b/installation/sql/mysql/joomla.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS `#__assets` ( UNIQUE KEY `idx_asset_name` (`name`), KEY `idx_lft_rgt` (`lft`,`rgt`), KEY `idx_parent_id` (`parent_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -- Dumping data for table `#__assets` @@ -95,7 +95,7 @@ CREATE TABLE IF NOT EXISTS `#__associations` ( `key` char(32) NOT NULL COMMENT 'The key for the association computed from an md5 on associated ids.', PRIMARY KEY (`context`,`id`), KEY `idx_key` (`key`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -108,7 +108,7 @@ CREATE TABLE IF NOT EXISTS `#__banners` ( `cid` int(11) NOT NULL DEFAULT 0, `type` int(11) NOT NULL DEFAULT 0, `name` varchar(255) NOT NULL DEFAULT '', - `alias` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `alias` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', `imptotal` int(11) NOT NULL DEFAULT 0, `impmade` int(11) NOT NULL DEFAULT 0, `clicks` int(11) NOT NULL DEFAULT 0, @@ -144,7 +144,7 @@ CREATE TABLE IF NOT EXISTS `#__banners` ( KEY `idx_metakey_prefix` (`metakey_prefix`), KEY `idx_banner_catid` (`catid`), KEY `idx_language` (`language`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -170,7 +170,7 @@ CREATE TABLE IF NOT EXISTS `#__banner_clients` ( PRIMARY KEY (`id`), KEY `idx_own_prefix` (`own_prefix`), KEY `idx_metakey_prefix` (`metakey_prefix`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -187,7 +187,7 @@ CREATE TABLE IF NOT EXISTS `#__banner_tracks` ( KEY `idx_track_date` (`track_date`), KEY `idx_track_type` (`track_type`), KEY `idx_banner_id` (`banner_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -205,7 +205,7 @@ CREATE TABLE IF NOT EXISTS `#__categories` ( `path` varchar(255) NOT NULL DEFAULT '', `extension` varchar(50) NOT NULL DEFAULT '', `title` varchar(255) NOT NULL, - `alias` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `alias` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', `note` varchar(255) NOT NULL DEFAULT '', `description` mediumtext NOT NULL, `published` tinyint(1) NOT NULL DEFAULT 0, @@ -231,7 +231,7 @@ CREATE TABLE IF NOT EXISTS `#__categories` ( KEY `idx_left_right` (`lft`,`rgt`), KEY `idx_alias` (`alias`), KEY `idx_language` (`language`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -- Dumping data for table `#__categories` @@ -254,7 +254,7 @@ INSERT INTO `#__categories` (`id`, `asset_id`, `parent_id`, `lft`, `rgt`, `level CREATE TABLE IF NOT EXISTS `#__contact_details` ( `id` int(11) NOT NULL AUTO_INCREMENT, `name` varchar(255) NOT NULL DEFAULT '', - `alias` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `alias` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', `con_position` varchar(255) DEFAULT NULL, `address` text, `suburb` varchar(100) DEFAULT NULL, @@ -304,7 +304,7 @@ CREATE TABLE IF NOT EXISTS `#__contact_details` ( KEY `idx_featured_catid` (`featured`,`catid`), KEY `idx_language` (`language`), KEY `idx_xreference` (`xreference`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -316,7 +316,7 @@ CREATE TABLE IF NOT EXISTS `#__content` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `asset_id` int(10) unsigned NOT NULL DEFAULT 0 COMMENT 'FK to the #__assets table.', `title` varchar(255) NOT NULL DEFAULT '', - `alias` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `alias` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', `introtext` mediumtext NOT NULL, `fulltext` mediumtext NOT NULL, `state` tinyint(3) NOT NULL DEFAULT 0, @@ -352,7 +352,7 @@ CREATE TABLE IF NOT EXISTS `#__content` ( KEY `idx_featured_catid` (`featured`,`catid`), KEY `idx_language` (`language`), KEY `idx_xreference` (`xreference`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -364,7 +364,7 @@ CREATE TABLE IF NOT EXISTS `#__content_frontpage` ( `content_id` int(11) NOT NULL DEFAULT 0, `ordering` int(11) NOT NULL DEFAULT 0, PRIMARY KEY (`content_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -378,7 +378,7 @@ CREATE TABLE IF NOT EXISTS `#__content_rating` ( `rating_count` int(10) unsigned NOT NULL DEFAULT 0, `lastip` varchar(50) NOT NULL DEFAULT '', PRIMARY KEY (`content_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -397,7 +397,7 @@ CREATE TABLE IF NOT EXISTS `#__content_types` ( `content_history_options` varchar(5120) COMMENT 'JSON string for com_contenthistory options', PRIMARY KEY (`type_id`), KEY `idx_alias` (`type_alias`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=10000; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci AUTO_INCREMENT=10000; -- -- Dumping data for table `#__content_types` @@ -437,7 +437,7 @@ CREATE TABLE IF NOT EXISTS `#__contentitem_tag_map` ( KEY `idx_tag` (`tag_id`), KEY `idx_type` (`type_id`), KEY `idx_core_content_id` (`core_content_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Maps items from content tables to tags'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci COMMENT='Maps items from content tables to tags'; -- -------------------------------------------------------- @@ -448,7 +448,7 @@ CREATE TABLE IF NOT EXISTS `#__contentitem_tag_map` ( CREATE TABLE IF NOT EXISTS `#__core_log_searches` ( `search_term` varchar(128) NOT NULL DEFAULT '', `hits` int(10) unsigned NOT NULL DEFAULT 0 -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -478,7 +478,7 @@ CREATE TABLE IF NOT EXISTS `#__extensions` ( KEY `element_clientid` (`element`,`client_id`), KEY `element_folder_clientid` (`element`,`folder`,`client_id`), KEY `extension` (`type`,`element`,`folder`,`client_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=10000; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci AUTO_INCREMENT=10000; -- -- Dumping data for table `#__extensions` @@ -641,7 +641,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_filters` ( `data` text NOT NULL, `params` mediumtext, PRIMARY KEY (`filter_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -676,7 +676,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links` ( KEY `idx_url` (`url`(75)), KEY `idx_published_list` (`published`,`state`,`access`,`publish_start_date`,`publish_end_date`,`list_price`), KEY `idx_published_sale` (`published`,`state`,`access`,`publish_start_date`,`publish_end_date`,`sale_price`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -691,7 +691,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_terms0` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -706,7 +706,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_terms1` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -721,7 +721,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_terms2` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -736,7 +736,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_terms3` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -751,7 +751,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_terms4` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -766,7 +766,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_terms5` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -781,7 +781,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_terms6` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -796,7 +796,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_terms7` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -811,7 +811,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_terms8` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -826,7 +826,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_terms9` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -841,7 +841,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_termsa` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -856,7 +856,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_termsb` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -871,7 +871,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_termsc` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -886,7 +886,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_termsd` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -901,7 +901,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_termse` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -916,7 +916,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_links_termsf` ( PRIMARY KEY (`link_id`,`term_id`), KEY `idx_term_weight` (`term_id`,`weight`), KEY `idx_link_term_weight` (`link_id`,`term_id`,`weight`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -937,7 +937,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_taxonomy` ( KEY `ordering` (`ordering`), KEY `access` (`access`), KEY `idx_parent_published` (`parent_id`,`state`,`access`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -- Dumping data for table `#__finder_taxonomy` @@ -958,7 +958,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_taxonomy_map` ( PRIMARY KEY (`link_id`,`node_id`), KEY `link_id` (`link_id`), KEY `node_id` (`node_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -981,7 +981,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_terms` ( KEY `idx_term_phrase` (`term`,`phrase`), KEY `idx_stem_phrase` (`stem`,`phrase`), KEY `idx_soundex_phrase` (`soundex`,`phrase`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -994,7 +994,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_terms_common` ( `language` varchar(3) NOT NULL, KEY `idx_word_lang` (`term`,`language`), KEY `idx_lang` (`language`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -- Dumping data for table `#__finder_terms_common` @@ -1133,7 +1133,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_tokens` ( `language` char(3) NOT NULL DEFAULT '', KEY `idx_word` (`term`), KEY `idx_context` (`context`) -) ENGINE=MEMORY DEFAULT CHARSET=utf8; +) ENGINE=MEMORY DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1155,7 +1155,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_tokens_aggregate` ( `language` char(3) NOT NULL DEFAULT '', KEY `token` (`term`), KEY `keyword_id` (`term_id`) -) ENGINE=MEMORY DEFAULT CHARSET=utf8; +) ENGINE=MEMORY DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1169,7 +1169,7 @@ CREATE TABLE IF NOT EXISTS `#__finder_types` ( `mime` varchar(100) NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `title` (`title`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1197,7 +1197,7 @@ CREATE TABLE IF NOT EXISTS `#__languages` ( UNIQUE KEY `idx_langcode` (`lang_code`), KEY `idx_access` (`access`), KEY `idx_ordering` (`ordering`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -- Dumping data for table `#__languages` @@ -1216,7 +1216,7 @@ CREATE TABLE IF NOT EXISTS `#__menu` ( `id` int(11) NOT NULL AUTO_INCREMENT, `menutype` varchar(24) NOT NULL COMMENT 'The type of menu this item belongs to. FK to #__menu_types.menutype', `title` varchar(255) NOT NULL COMMENT 'The display title of the menu item.', - `alias` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'The SEF alias of the menu item.', + `alias` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'The SEF alias of the menu item.', `note` varchar(255) NOT NULL DEFAULT '', `path` varchar(1024) NOT NULL COMMENT 'The computed path of the menu item based on the alias field.', `link` varchar(1024) NOT NULL COMMENT 'The actually link the menu item refers to.', @@ -1238,14 +1238,14 @@ CREATE TABLE IF NOT EXISTS `#__menu` ( `language` char(7) NOT NULL DEFAULT '', `client_id` tinyint(4) NOT NULL DEFAULT 0, PRIMARY KEY (`id`), - UNIQUE KEY `idx_client_id_parent_id_alias_language` (`client_id`,`parent_id`,`alias`,`language`), + UNIQUE KEY `idx_client_id_parent_id_alias_language` (`client_id`,`parent_id`,`alias`(191),`language`), KEY `idx_componentid` (`component_id`,`menutype`,`published`,`access`), KEY `idx_menutype` (`menutype`), KEY `idx_left_right` (`lft`,`rgt`), KEY `idx_alias` (`alias`), - KEY `idx_path` (`path`(255)), + KEY `idx_path` (`path`(191)), KEY `idx_language` (`language`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=102; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci AUTO_INCREMENT=102; -- -- Dumping data for table `#__menu` @@ -1288,7 +1288,7 @@ CREATE TABLE IF NOT EXISTS `#__menu_types` ( `description` varchar(255) NOT NULL DEFAULT '', PRIMARY KEY (`id`), UNIQUE KEY `idx_menutype` (`menutype`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -- Dumping data for table `#__menu_types` @@ -1315,7 +1315,7 @@ CREATE TABLE IF NOT EXISTS `#__messages` ( `message` text NOT NULL, PRIMARY KEY (`message_id`), KEY `useridto_state` (`user_id_to`,`state`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1328,7 +1328,7 @@ CREATE TABLE IF NOT EXISTS `#__messages_cfg` ( `cfg_name` varchar(100) NOT NULL DEFAULT '', `cfg_value` varchar(255) NOT NULL DEFAULT '', UNIQUE KEY `idx_user_var_name` (`user_id`,`cfg_name`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1359,7 +1359,7 @@ CREATE TABLE IF NOT EXISTS `#__modules` ( KEY `published` (`published`,`access`), KEY `newsfeeds` (`module`,`published`), KEY `idx_language` (`language`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=87; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci AUTO_INCREMENT=87; -- -- Dumping data for table `#__modules` @@ -1392,7 +1392,7 @@ CREATE TABLE IF NOT EXISTS `#__modules_menu` ( `moduleid` int(11) NOT NULL DEFAULT 0, `menuid` int(11) NOT NULL DEFAULT 0, PRIMARY KEY (`moduleid`,`menuid`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -- Dumping data for table `#__modules_menu` @@ -1427,7 +1427,7 @@ CREATE TABLE IF NOT EXISTS `#__newsfeeds` ( `catid` int(11) NOT NULL DEFAULT 0, `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `name` varchar(100) NOT NULL DEFAULT '', - `alias` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `alias` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', `link` varchar(200) NOT NULL DEFAULT '', `published` tinyint(1) NOT NULL DEFAULT 0, `numarticles` int(10) unsigned NOT NULL DEFAULT 1, @@ -1462,7 +1462,7 @@ CREATE TABLE IF NOT EXISTS `#__newsfeeds` ( KEY `idx_createdby` (`created_by`), KEY `idx_language` (`language`), KEY `idx_xreference` (`xreference`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1476,7 +1476,7 @@ CREATE TABLE IF NOT EXISTS `#__overrider` ( `string` text NOT NULL, `file` varchar(255) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1500,7 +1500,7 @@ CREATE TABLE IF NOT EXISTS `#__postinstall_messages` ( `version_introduced` varchar(50) NOT NULL DEFAULT '3.2.0' COMMENT 'Version when this message was introduced', `enabled` tinyint(3) NOT NULL DEFAULT 1, PRIMARY KEY (`postinstall_message_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; INSERT INTO `#__postinstall_messages` (`extension_id`, `title_key`, `description_key`, `action_key`, `language_extension`, `language_client_id`, `type`, `action_file`, `action`, `condition_file`, `condition_method`, `version_introduced`, `enabled`) VALUES (700, 'PLG_TWOFACTORAUTH_TOTP_POSTINSTALL_TITLE', 'PLG_TWOFACTORAUTH_TOTP_POSTINSTALL_BODY', 'PLG_TWOFACTORAUTH_TOTP_POSTINSTALL_ACTION', 'plg_twofactorauth_totp', 1, 'action', 'site://plugins/twofactorauth/totp/postinstall/actions.php', 'twofactorauth_postinstall_action', 'site://plugins/twofactorauth/totp/postinstall/actions.php', 'twofactorauth_postinstall_condition', '3.2.0', 1), @@ -1524,9 +1524,9 @@ CREATE TABLE IF NOT EXISTS `#__redirect_links` ( `modified_date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `header` smallint(3) NOT NULL DEFAULT 301, PRIMARY KEY (`id`), - UNIQUE KEY `idx_link_old` (`old_url`), + UNIQUE KEY `idx_link_old` (`old_url`(191)), KEY `idx_link_modifed` (`modified_date`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1538,7 +1538,7 @@ CREATE TABLE IF NOT EXISTS `#__schemas` ( `extension_id` int(11) NOT NULL, `version_id` varchar(20) NOT NULL, PRIMARY KEY (`extension_id`,`version_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1547,7 +1547,7 @@ CREATE TABLE IF NOT EXISTS `#__schemas` ( -- CREATE TABLE IF NOT EXISTS `#__session` ( - `session_id` varchar(200) NOT NULL DEFAULT '', + `session_id` varchar(191) NOT NULL DEFAULT '', `client_id` tinyint(3) unsigned NOT NULL DEFAULT 0, `guest` tinyint(4) unsigned DEFAULT 1, `time` varchar(14) DEFAULT '', @@ -1557,7 +1557,7 @@ CREATE TABLE IF NOT EXISTS `#__session` ( PRIMARY KEY (`session_id`), KEY `userid` (`userid`), KEY `time` (`time`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1573,7 +1573,7 @@ CREATE TABLE IF NOT EXISTS `#__tags` ( `level` int(10) unsigned NOT NULL DEFAULT 0, `path` varchar(255) NOT NULL DEFAULT '', `title` varchar(255) NOT NULL, - `alias` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `alias` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', `note` varchar(255) NOT NULL DEFAULT '', `description` mediumtext NOT NULL, `published` tinyint(1) NOT NULL DEFAULT 0, @@ -1604,7 +1604,7 @@ CREATE TABLE IF NOT EXISTS `#__tags` ( KEY `idx_left_right` (`lft`,`rgt`), KEY `idx_alias` (`alias`), KEY `idx_language` (`language`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -- Dumping data for table `#__tags` @@ -1629,7 +1629,7 @@ CREATE TABLE IF NOT EXISTS `#__template_styles` ( PRIMARY KEY (`id`), KEY `idx_template` (`template`), KEY `idx_home` (`home`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=9; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci AUTO_INCREMENT=9; -- -- Dumping data for table `#__template_styles` @@ -1656,7 +1656,7 @@ CREATE TABLE IF NOT EXISTS `#__ucm_base` ( KEY `idx_ucm_item_id` (`ucm_item_id`), KEY `idx_ucm_type_id` (`ucm_type_id`), KEY `idx_ucm_language_id` (`ucm_language_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1668,7 +1668,7 @@ CREATE TABLE IF NOT EXISTS `#__ucm_content` ( `core_content_id` int(10) unsigned NOT NULL AUTO_INCREMENT, `core_type_alias` varchar(255) NOT NULL DEFAULT '' COMMENT 'FK to the content types table', `core_title` varchar(255) NOT NULL, - `core_alias` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `core_alias` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT '', `core_body` mediumtext NOT NULL, `core_state` tinyint(1) NOT NULL DEFAULT 0, `core_checked_out_time` varchar(255) NOT NULL DEFAULT '', @@ -1710,7 +1710,7 @@ CREATE TABLE IF NOT EXISTS `#__ucm_content` ( KEY `idx_core_checked_out_user_id` (`core_checked_out_user_id`), KEY `idx_core_created_user_id` (`core_created_user_id`), KEY `idx_core_type_id` (`core_type_id`) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Contains core content data in name spaced fields'; + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci COMMENT='Contains core content data in name spaced fields'; -- -------------------------------------------------------- @@ -1732,7 +1732,7 @@ CREATE TABLE IF NOT EXISTS `#__ucm_history` ( PRIMARY KEY (`version_id`), KEY `idx_ucm_item_id` (`ucm_type_id`,`ucm_item_id`), KEY `idx_save_date` (`save_date`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1756,7 +1756,7 @@ CREATE TABLE IF NOT EXISTS `#__updates` ( `infourl` text NOT NULL, `extra_query` varchar(1000) DEFAULT '', PRIMARY KEY (`update_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Available Updates'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci COMMENT='Available Updates'; -- -------------------------------------------------------- @@ -1773,7 +1773,7 @@ CREATE TABLE IF NOT EXISTS `#__update_sites` ( `last_check_timestamp` bigint(20) DEFAULT 0, `extra_query` varchar(1000) DEFAULT '', PRIMARY KEY (`update_site_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Update Sites'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci COMMENT='Update Sites'; -- -- Dumping data for table `#__update_sites` @@ -1795,7 +1795,7 @@ CREATE TABLE IF NOT EXISTS `#__update_sites_extensions` ( `update_site_id` int(11) NOT NULL DEFAULT 0, `extension_id` int(11) NOT NULL DEFAULT 0, PRIMARY KEY (`update_site_id`,`extension_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Links extensions to update sites'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci COMMENT='Links extensions to update sites'; -- -- Dumping data for table `#__update_sites_extensions` @@ -1824,7 +1824,7 @@ CREATE TABLE IF NOT EXISTS `#__usergroups` ( KEY `idx_usergroup_title_lookup` (`title`), KEY `idx_usergroup_adjacency_lookup` (`parent_id`), KEY `idx_usergroup_nested_set_lookup` (`lft`,`rgt`) USING BTREE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -- Dumping data for table `#__usergroups` @@ -1869,7 +1869,7 @@ CREATE TABLE IF NOT EXISTS `#__users` ( KEY `idx_block` (`block`), KEY `username` (`username`), KEY `email` (`email`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1881,7 +1881,7 @@ CREATE TABLE IF NOT EXISTS `#__user_keys` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, `user_id` varchar(255) NOT NULL, `token` varchar(255) NOT NULL, - `series` varchar(255) NOT NULL, + `series` varchar(191) NOT NULL, `invalid` tinyint(4) NOT NULL, `time` varchar(200) NOT NULL, `uastring` varchar(255) NOT NULL, @@ -1890,7 +1890,7 @@ CREATE TABLE IF NOT EXISTS `#__user_keys` ( UNIQUE KEY `series_2` (`series`), UNIQUE KEY `series_3` (`series`), KEY `user_id` (`user_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1917,7 +1917,7 @@ CREATE TABLE IF NOT EXISTS `#__user_notes` ( PRIMARY KEY (`id`), KEY `idx_user_id` (`user_id`), KEY `idx_category_id` (`catid`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1931,7 +1931,7 @@ CREATE TABLE IF NOT EXISTS `#__user_profiles` ( `profile_value` text NOT NULL, `ordering` int(11) NOT NULL DEFAULT 0, UNIQUE KEY `idx_user_id_profile_key` (`user_id`,`profile_key`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT='Simple user profile storage table'; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci COMMENT='Simple user profile storage table'; -- -------------------------------------------------------- @@ -1943,7 +1943,7 @@ CREATE TABLE IF NOT EXISTS `#__user_usergroup_map` ( `user_id` int(10) unsigned NOT NULL DEFAULT 0 COMMENT 'Foreign Key to #__users.id', `group_id` int(10) unsigned NOT NULL DEFAULT 0 COMMENT 'Foreign Key to #__usergroups.id', PRIMARY KEY (`user_id`,`group_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci; -- -------------------------------------------------------- @@ -1958,7 +1958,7 @@ CREATE TABLE IF NOT EXISTS `#__viewlevels` ( `rules` varchar(5120) NOT NULL COMMENT 'JSON encoded access control.', PRIMARY KEY (`id`), UNIQUE KEY `idx_assetgroup_title_lookup` (`title`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 AUTO_INCREMENT=7; +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DEFAULT COLLATE=utf8mb4_general_ci AUTO_INCREMENT=7; -- -- Dumping data for table `#__viewlevels` diff --git a/libraries/cms/installer/installer.php b/libraries/cms/installer/installer.php index eca2c47d8eb3e..ffd0e5b18d4dc 100644 --- a/libraries/cms/installer/installer.php +++ b/libraries/cms/installer/installer.php @@ -945,6 +945,23 @@ public function parseSQLFiles($element) if ($query != '' && $query{0} != '#') { + /** + * If we don't have UTF-8 Multibyte support we'll have to convert queries to plain UTF-8 + * + * Note: the JDatabaseDriver::convertUtf8mb4QueryToUtf8 performs the conversion ONLY when + * necessary, so there's no need to check the conditions in JInstaller. + */ + $query = $db->convertUtf8mb4QueryToUtf8($query); + + /** + * This is a query which was supposed to convert tables to utf8mb4 charset but the server doesn't + * support utf8mb4. Therefore we don't have to run it, it has no effect and it's a mere waste of time. + */ + if (!$db->hasUTF8mb4Support() && stristr($query, 'CONVERT TO CHARACTER SET utf8 ')) + { + continue; + } + $db->setQuery($query); if (!$db->execute()) diff --git a/libraries/joomla/database/driver.php b/libraries/joomla/database/driver.php index 7e005a3a28d0e..d59f8d234819c 100644 --- a/libraries/joomla/database/driver.php +++ b/libraries/joomla/database/driver.php @@ -52,6 +52,15 @@ abstract class JDatabaseDriver extends JDatabase implements JDatabaseInterface */ public $name; + /** + * The type of the database server family supported by this driver. Examples: mysql, oracle, postgresql, mssql, + * sqlite. + * + * @var string + * @since CMS 3.5.0 + */ + public $serverType; + /** * @var resource The database connection resource. * @since 11.1 @@ -146,6 +155,12 @@ abstract class JDatabaseDriver extends JDatabase implements JDatabaseInterface */ protected $utf = true; + /** + * @var boolean True if the database engine supports UTF-8 Multibyte (utf8mb4) character encoding. + * @since CMS 3.5.0 + */ + protected $utf8mb4 = false; + /** * @var integer The database error number * @since 11.1 @@ -455,6 +470,98 @@ public function alterDbCharacterSet($dbName) return $this->execute(); } + /** + * Alter a table's character set, obtaining an array of queries to do so from a protected method. The conversion is + * wrapped in a transaction, if supported by the database driver. Otherwise the table will be locked before the + * conversion. This prevents data corruption. + * + * @param string $tableName The name of the table to alter + * @param boolean $rethrow True to rethrow database exceptions. Default: false (exceptions are suppressed) + * + * @return boolean True if successful + * + * @since CMS 3.5.0 + * @throws RuntimeException If the table name is empty + * @throws Exception Relayed from the database layer if a database error occurs and $rethrow == true + */ + public function alterTableCharacterSet($tableName, $rethrow = false) + { + if (is_null($tableName)) + { + throw new RuntimeException('Table name must not be null.'); + } + + $queries = $this->getAlterTableCharacterSet($tableName); + + if (empty($queries)) + { + return false; + } + + $hasTransaction = true; + + try + { + $this->transactionStart(); + } + catch (Exception $e) + { + $hasTransaction = false; + $this->lockTable($tableName); + } + + foreach ($queries as $query) + { + try + { + $this->setQuery($query)->execute(); + } + catch (Exception $e) + { + if ($hasTransaction) + { + $this->transactionRollback(); + } + else + { + $this->unlockTables(); + } + + if ($rethrow) + { + throw $e; + } + + return false; + } + } + + if ($hasTransaction) + { + try + { + $this->transactionCommit(); + } + catch (Exception $e) + { + $this->transactionRollback(); + + if ($rethrow) + { + throw $e; + } + + return false; + } + } + else + { + $this->unlockTables(); + } + + return true; + } + /** * Connects to the database if needed. * @@ -620,7 +727,124 @@ abstract public function getAffectedRows(); */ protected function getAlterDbCharacterSet($dbName) { - return 'ALTER DATABASE ' . $this->quoteName($dbName) . ' CHARACTER SET `utf8`'; + $charset = $this->utf8mb4 ? 'utf8mb4' : 'utf8'; + + return 'ALTER DATABASE ' . $this->quoteName($dbName) . ' CHARACTER SET `' . $charset . '`'; + } + + /** + * Get the query strings to alter the character set and collation of a table. + * + * @param string $tableName The name of the table + * + * @return string[] The queries required to alter the table's character set and collation + * + * @since CMS 3.5.0 + */ + public function getAlterTableCharacterSet($tableName) + { + $charset = $this->utf8mb4 ? 'utf8mb4' : 'utf8'; + $collation = $charset . '_general_ci'; + + $quotedTableName = $this->quoteName($tableName); + + $queries = array(); + $queries[] = "ALTER TABLE $quotedTableName CONVERT TO CHARACTER SET $charset COLLATE $collation"; + + /** + * We also need to convert each text column, modifying their character set and collation. This allows us to + * change, for example, a utf8_bin collated column to a utf8mb4_bin collated column. + */ + $sql = "SHOW FULL COLUMNS FROM $quotedTableName"; + $this->setQuery($sql); + $columns = $this->loadAssocList(); + $columnMods = array(); + + if (is_array($columns)) + { + foreach ($columns as $column) + { + // Make sure we are redefining only columns which do support a collation + $col = (object) $column; + + if (empty($col->Collation)) + { + continue; + } + + // Default new collation: utf8_general_ci or utf8mb4_general_ci + $newCollation = $charset . '_general_ci'; + $collationParts = explode('_', $col->Collation); + + /** + * If the collation is in the form charset_collationType_ci or charset_collationType we have to change + * the charset but leave the collationType intact (e.g. utf8_bin must become utf8mb4_bin, NOT + * utf8mb4_general_ci). + */ + if (count($collationParts) >= 2) + { + $ci = array_pop($collationParts); + $collationType = array_pop($collationParts); + $newCollation = $charset . '_' . $collationType . '_' . $ci; + + /** + * When the last part of the old collation is not _ci we have a charset_collationType format, + * something like utf8_bin. Therefore the new collation only has *two* parts. + */ + if ($ci != 'ci') + { + $newCollation = $charset . '_' . $ci; + } + } + + // If the old and new collation is the same we don't have to change the collation type + if (strtolower($newCollation) == strtolower($col->Collation)) + { + continue; + } + + $null = $col->Null == 'YES' ? 'NULL' : 'NOT NULL'; + $default = is_null($col->Default) ? '' : "DEFAULT '" . $this->q($col->Default) . "'"; + $columnMods[] = "MODIFY COLUMN `{$col->Field}` {$col->Type} CHARACTER SET $charset COLLATE $newCollation $null $default"; + } + } + + if (count($columnMods)) + { + $queries[] = "ALTER TABLE $quotedTableName " . + implode(',', $columnMods) . + " CHARACTER SET $charset COLLATE $collation"; + } + + return $queries; + } + + /** + * Automatically downgrade a CREATE TABLE or ALTER TABLE query from utf8mb4 (UTF-8 Multibyte) to plain utf8. Used + * when the server doesn't support UTF-8 Multibyte. + * + * @param string $query The query to convert + * + * @return string The converted query + */ + public function convertUtf8mb4QueryToUtf8($query) + { + if ($this->hasUTF8mb4Support()) + { + return $query; + } + + // If it's not an ALTER TABLE or CREATE TABLE command there's nothing to convert + $beginningOfQuery = substr($query, 0, 12); + $beginningOfQuery = strtoupper($beginningOfQuery); + + if (!in_array($beginningOfQuery, array('ALTER TABLE ', 'CREATE TABLE'))) + { + return $query; + } + + // Replace utf8mb4 with utf8 + return str_replace('utf8mb4', 'utf8', $query); } /** @@ -639,7 +863,9 @@ protected function getCreateDatabaseQuery($options, $utf) { if ($utf) { - return 'CREATE DATABASE ' . $this->quoteName($options->db_name) . ' CHARACTER SET `utf8`'; + $charset = $this->utf8mb4 ? 'utf8mb4' : 'utf8'; + + return 'CREATE DATABASE ' . $this->quoteName($options->db_name) . ' CHARACTER SET `' . $charset . '`'; } return 'CREATE DATABASE ' . $this->quoteName($options->db_name); @@ -654,6 +880,17 @@ protected function getCreateDatabaseQuery($options, $utf) */ abstract public function getCollation(); + /** + * Method to get the database connection collation, as reported by the driver. If the connector doesn't support + * reporting this value please return an empty string. + * + * @return string + */ + public function getConnectionCollation() + { + return ''; + } + /** * Method that provides access to the underlying database connection. Useful for when you need to call a * proprietary method such as postgresql's lo_* methods. @@ -838,6 +1075,74 @@ public function getImporter() return $o; } + /** + * Get the name of the database driver. If $this->name is not set it will try guessing the driver name from the + * class name. + * + * @return string + * + * @since CMS 3.5.0 + */ + public function getName() + { + if (empty($this->name)) + { + $className = get_class($this); + $className = str_replace('JDatabaseDriver', '', $className); + $this->name = strtolower($className); + } + + return $this->name; + } + + /** + * Get the server family type, e.g. mysql, postgresql, oracle, sqlite, mssql. If $this->serverType is not set it + * will attempt guessing the server family type from the driver name. If this is not possible the driver name will + * be returned instead. + * + * @return string + * + * @since CMS 3.5.0 + */ + public function getServerType() + { + if (empty($this->serverType)) + { + $name = $this->getName(); + + if (stristr($name, 'mysql') !== false) + { + $this->serverType = 'mysql'; + } + elseif (stristr($name, 'postgre') !== false) + { + $this->serverType = 'postgresql'; + } + elseif (stristr($name, 'oracle') !== false) + { + $this->serverType = 'oracle'; + } + elseif (stristr($name, 'sqlite') !== false) + { + $this->serverType = 'sqlite'; + } + elseif (stristr($name, 'sqlsrv') !== false) + { + $this->serverType = 'mssql'; + } + elseif (stristr($name, 'mssql') !== false) + { + $this->serverType = 'mssql'; + } + else + { + $this->serverType = $name; + } + } + + return $this->serverType; + } + /** * Get the current query object or a new JDatabaseQuery object. * @@ -971,6 +1276,19 @@ public function hasUTFSupport() return $this->utf; } + /** + * Determine whether the database engine support the UTF-8 Multibyte (utf8mb4) character encoding. This applies to + * MySQL databases. + * + * @return boolean True if the database engine supports UTF-8 Multibyte. + * + * @since CMS 3.5.0 + */ + public function hasUTF8mb4Support() + { + return $this->utf8mb4; + } + /** * Get the version of the database connector * diff --git a/libraries/joomla/database/driver/mysql.php b/libraries/joomla/database/driver/mysql.php index 11bf5a5f7a721..13c259acb53ba 100644 --- a/libraries/joomla/database/driver/mysql.php +++ b/libraries/joomla/database/driver/mysql.php @@ -100,8 +100,11 @@ public function connect() $this->select($this->options['database']); } - // Set charactersets (needed for MySQL 4.1.2+). - $this->setUTF(); + // Pre-populate the UTF-8 Multibyte compatibility flag based on server version + $this->utf8mb4 = $this->serverClaimsUtf8mb4Support(); + + // Set the character set (needed for MySQL 4.1.2+). + $this->utf = $this->setUTF(); // Turn MySQL profiling ON in debug mode: if ($this->debug && $this->hasProfiling()) @@ -380,9 +383,34 @@ public function select($database) */ public function setUTF() { + // If UTF is not supported return false immediately + if (!$this->utf) + { + return false; + } + + // Make sure we're connected to the server $this->connect(); - return mysql_set_charset('utf8', $this->connection); + // Which charset should I use, plain utf8 or multibyte utf8mb4? + $charset = $this->utf8mb4 ? 'utf8mb4' : 'utf8'; + + $result = @mysql_set_charset($charset, $this->connection); + + /** + * If I could not set the utf8mb4 charset then the server doesn't support utf8mb4 despite claiming otherwise. + * This happens on old MySQL server versions (less than 5.5.3) using the mysqlnd PHP driver. Since mysqlnd + * masks the server version and reports only its own we can not be sure if the server actually does support + * UTF-8 Multibyte (i.e. it's MySQL 5.5.3 or later). Since the utf8mb4 charset is undefined in this case we + * catch the error and determine that utf8mb4 is not supported! + */ + if (!$result && $this->utf8mb4) + { + $this->utf8mb4 = false; + $result = @mysql_set_charset('utf8', $this->connection); + } + + return $result; } /** @@ -463,4 +491,29 @@ private function hasProfiling() return false; } } + + /** + * Does the database server claim to have support for UTF-8 Multibyte (utf8mb4) collation? + * + * libmysql supports utf8mb4 since 5.5.3 (same version as the MySQL server). mysqlnd supports utf8mb4 since 5.0.9. + * + * @return boolean + * + * @since CMS 3.5.0 + */ + private function serverClaimsUtf8mb4Support() + { + $client_version = mysql_get_client_info(); + + if (strpos($client_version, 'mysqlnd') !== false) + { + $client_version = preg_replace('/^\D+([\d.]+).*/', '$1', $client_version); + + return version_compare($client_version, '5.0.9', '>='); + } + else + { + return version_compare($client_version, '5.5.3', '>='); + } + } } diff --git a/libraries/joomla/database/driver/mysqli.php b/libraries/joomla/database/driver/mysqli.php index e3fa8a0cf9c6a..9faaa4a18a329 100644 --- a/libraries/joomla/database/driver/mysqli.php +++ b/libraries/joomla/database/driver/mysqli.php @@ -25,6 +25,20 @@ class JDatabaseDriverMysqli extends JDatabaseDriver */ public $name = 'mysqli'; + /** + * The type of the database server family supported by this driver. + * + * @var string + * @since CMS 3.5.0 + */ + public $serverType = 'mysql'; + + /** + * @var mysqli The database connection resource. + * @since 11.1 + */ + protected $connection; + /** * The character(s) used to quote SQL statement names such as table names or field names, * etc. The child classes should define this as necessary. If a single character string the @@ -178,8 +192,11 @@ public function connect() $this->select($this->options['database']); } - // Set charactersets (needed for MySQL 4.1.2+). - $this->setUTF(); + // Pre-populate the UTF-8 Multibyte compatibility flag based on server version + $this->utf8mb4 = $this->serverClaimsUtf8mb4Support(); + + // Set the character set (needed for MySQL 4.1.2+). + $this->utf = $this->setUTF(); // Turn MySQL profiling ON in debug mode: if ($this->debug && $this->hasProfiling()) @@ -329,6 +346,30 @@ public function getCollation() } } + /** + * Method to get the database connection collation, as reported by the driver. If the connector doesn't support + * reporting this value please return an empty string. + * + * @return string + */ + public function getConnectionCollation() + { + $this->connect(); + + // Attempt to get the database collation by accessing the server system variable. + $this->setQuery('SHOW VARIABLES LIKE "collation_connection"'); + $result = $this->loadObject(); + + if (property_exists($result, 'Value')) + { + return $result->Value; + } + else + { + return false; + } + } + /** * Get the number of returned rows for the previous executed SQL statement. * @@ -670,9 +711,34 @@ public function select($database) */ public function setUTF() { + // If UTF is not supported return false immediately + if (!$this->utf) + { + return false; + } + + // Make sure we're connected to the server $this->connect(); - return $this->connection->set_charset('utf8'); + // Which charset should I use, plain utf8 or multibyte utf8mb4? + $charset = $this->utf8mb4 ? 'utf8mb4' : 'utf8'; + + $result = @$this->connection->set_charset($charset); + + /** + * If I could not set the utf8mb4 charset then the server doesn't support utf8mb4 despite claiming otherwise. + * This happens on old MySQL server versions (less than 5.5.3) using the mysqlnd PHP driver. Since mysqlnd + * masks the server version and reports only its own we can not be sure if the server actually does support + * UTF-8 Multibyte (i.e. it's MySQL 5.5.3 or later). Since the utf8mb4 charset is undefined in this case we + * catch the error and determine that utf8mb4 is not supported! + */ + if (!$result && $this->utf8mb4) + { + $this->utf8mb4 = false; + $result = @$this->connection->set_charset('utf8'); + } + + return $result; } /** @@ -866,4 +932,29 @@ private function hasProfiling() return false; } } + + /** + * Does the database server claim to have support for UTF-8 Multibyte (utf8mb4) collation? + * + * libmysql supports utf8mb4 since 5.5.3 (same version as the MySQL server). mysqlnd supports utf8mb4 since 5.0.9. + * + * @return boolean + * + * @since CMS 3.5.0 + */ + private function serverClaimsUtf8mb4Support() + { + $client_version = mysqli_get_client_info(); + + if (strpos($client_version, 'mysqlnd') !== false) + { + $client_version = preg_replace('/^\D+([\d.]+).*/', '$1', $client_version); + + return version_compare($client_version, '5.0.9', '>='); + } + else + { + return version_compare($client_version, '5.5.3', '>='); + } + } } diff --git a/libraries/joomla/database/driver/oracle.php b/libraries/joomla/database/driver/oracle.php index 2d0ef041ad170..2101cc86092d2 100644 --- a/libraries/joomla/database/driver/oracle.php +++ b/libraries/joomla/database/driver/oracle.php @@ -25,6 +25,14 @@ class JDatabaseDriverOracle extends JDatabaseDriverPdo */ public $name = 'oracle'; + /** + * The type of the database server family supported by this driver. + * + * @var string + * @since CMS 3.5.0 + */ + public $serverType = 'oracle'; + /** * The character(s) used to quote SQL statement names such as table names or field names, * etc. The child classes should define this as necessary. If a single character string the @@ -161,6 +169,17 @@ public function getCollation() return $this->charset; } + /** + * Method to get the database connection collation, as reported by the driver. If the connector doesn't support + * reporting this value please return an empty string. + * + * @return string + */ + public function getConnectionCollation() + { + return $this->charset; + } + /** * Get a query to run and verify the database is operational. * diff --git a/libraries/joomla/database/driver/pdo.php b/libraries/joomla/database/driver/pdo.php index 20a448da35386..837d992996f1a 100644 --- a/libraries/joomla/database/driver/pdo.php +++ b/libraries/joomla/database/driver/pdo.php @@ -25,6 +25,12 @@ abstract class JDatabaseDriverPdo extends JDatabaseDriver */ public $name = 'pdo'; + /** + * @var PDO The database connection resource. + * @since 12.1 + */ + protected $connection; + /** * The character(s) used to quote SQL statement names such as table names or field names, * etc. The child classes should define this as necessary. If a single character string the diff --git a/libraries/joomla/database/driver/pdomysql.php b/libraries/joomla/database/driver/pdomysql.php index d4dc6c77cf7ce..01269cbe57177 100644 --- a/libraries/joomla/database/driver/pdomysql.php +++ b/libraries/joomla/database/driver/pdomysql.php @@ -27,6 +27,14 @@ class JDatabaseDriverPdomysql extends JDatabaseDriverPdo */ public $name = 'pdomysql'; + /** + * The type of the database server family supported by this driver. + * + * @var string + * @since CMS 3.5.0 + */ + public $serverType = 'mysql'; + /** * The character(s) used to quote SQL statement names such as table names or field names, * etc. The child classes should define this as necessary. If a single character string the @@ -64,10 +72,23 @@ class JDatabaseDriverPdomysql extends JDatabaseDriverPdo */ public function __construct($options) { + /** + * Pre-populate the UTF-8 Multibyte compatibility flag. Unfortuantely PDO won't report the server version + * unless we're connected to it and we cannot connect to it unless we know if it supports utf8mb4 which requires + * us knowing the server version. Between this chicken and egg issue we _assume_ it's supported and we'll just + * catch any problems at connection time. + */ + $this->utf8mb4 = true; + // Get some basic values from the options. $options['driver'] = 'mysql'; $options['charset'] = (isset($options['charset'])) ? $options['charset'] : 'utf8'; + if ($this->utf8mb4 && ($options['charset'] == 'utf8')) + { + $options['charset'] = 'utf8mb4'; + } + $this->charset = $options['charset']; // Finalize initialisation. @@ -84,7 +105,33 @@ public function __construct($options) */ public function connect() { - parent::connect(); + try + { + // Try to connect to MySQL + parent::connect(); + } + catch (\RuntimeException $e) + { + // If the connection failed but not because of the wrong character set bubble up the exception + if (!$this->utf8mb4 || ($this->options['charset'] != 'utf8mb4')) + { + throw $e; + } + + /** + * If the connection failed and I was trying to use the utf8mb4 charset then it is likely that the server + * doesn't support utf8mb4 despite claiming otherwise. + * + * This happens on old MySQL server versions (less than 5.5.3) using the mysqlnd PHP driver. Since mysqlnd + * masks the server version and reports only its own we can not be sure if the server actually does support + * UTF-8 Multibyte (i.e. it's MySQL 5.5.3 or later). Since the utf8mb4 charset is undefined in this case we + * catch the error and determine that utf8mb4 is not supported! + */ + $this->utf8mb4 = false; + $this->options['charset'] = 'utf8'; + + parent::connect(); + } $this->connection->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); $this->connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, true); @@ -175,6 +222,30 @@ public function getCollation() } } + /** + * Method to get the database connection collation, as reported by the driver. If the connector doesn't support + * reporting this value please return an empty string. + * + * @return string + */ + public function getConnectionCollation() + { + $this->connect(); + + // Attempt to get the database collation by accessing the server system variable. + $this->setQuery('SHOW VARIABLES LIKE "collation_connection"'); + $result = $this->loadObject(); + + if (property_exists($result, 'Value')) + { + return $result->Value; + } + else + { + return false; + } + } + /** * Shows the table CREATE statement that creates the given tables. * diff --git a/libraries/joomla/database/driver/postgresql.php b/libraries/joomla/database/driver/postgresql.php index 67bc47b6076d2..1f86ede06e12a 100644 --- a/libraries/joomla/database/driver/postgresql.php +++ b/libraries/joomla/database/driver/postgresql.php @@ -24,6 +24,14 @@ class JDatabaseDriverPostgresql extends JDatabaseDriver */ public $name = 'postgresql'; + /** + * The type of the database server family supported by this driver. + * + * @var string + * @since CMS 3.5.0 + */ + public $serverType = 'postgresql'; + /** * Quote for named objects * @@ -265,6 +273,17 @@ public function getCollation() return $array[0]['lc_collate']; } + /** + * Method to get the database connection collation, as reported by the driver. If the connector doesn't support + * reporting this value please return an empty string. + * + * @return string + */ + public function getConnectionCollation() + { + return pg_client_encoding($this->connection); + } + /** * Get the number of returned rows for the previous executed SQL statement. * diff --git a/libraries/joomla/database/driver/sqlite.php b/libraries/joomla/database/driver/sqlite.php index 60dc93cee9255..9cbbf05c7dc64 100644 --- a/libraries/joomla/database/driver/sqlite.php +++ b/libraries/joomla/database/driver/sqlite.php @@ -25,6 +25,14 @@ class JDatabaseDriverSqlite extends JDatabaseDriverPdo */ public $name = 'sqlite'; + /** + * The type of the database server family supported by this driver. + * + * @var string + * @since CMS 3.5.0 + */ + public $serverType = 'sqlite'; + /** * The character(s) used to quote SQL statement names such as table names or field names, * etc. The child classes should define this as necessary. If a single character string the @@ -118,6 +126,17 @@ public function getCollation() return $this->charset; } + /** + * Method to get the database connection collation, as reported by the driver. If the connector doesn't support + * reporting this value please return an empty string. + * + * @return string + */ + public function getConnectionCollation() + { + return $this->charset; + } + /** * Shows the table CREATE statement that creates the given tables. * diff --git a/libraries/joomla/database/driver/sqlsrv.php b/libraries/joomla/database/driver/sqlsrv.php index 4a6664726b65d..f84c6546d871d 100644 --- a/libraries/joomla/database/driver/sqlsrv.php +++ b/libraries/joomla/database/driver/sqlsrv.php @@ -25,6 +25,14 @@ class JDatabaseDriverSqlsrv extends JDatabaseDriver */ public $name = 'sqlsrv'; + /** + * The type of the database server family supported by this driver. + * + * @var string + * @since CMS 3.5.0 + */ + public $serverType = 'mssql'; + /** * The character(s) used to quote SQL statement names such as table names or field names, * etc. The child classes should define this as necessary. If a single character string the @@ -309,6 +317,18 @@ public function getCollation() return 'MSSQL UTF-8 (UCS2)'; } + /** + * Method to get the database connection collation, as reported by the driver. If the connector doesn't support + * reporting this value please return an empty string. + * + * @return string + */ + public function getConnectionCollation() + { + // TODO: Not fake this + return 'MSSQL UTF-8 (UCS2)'; + } + /** * Get the number of returned rows for the previous executed SQL statement. * diff --git a/libraries/joomla/filter/input.php b/libraries/joomla/filter/input.php index 821055051d22a..e5db152ae3421 100644 --- a/libraries/joomla/filter/input.php +++ b/libraries/joomla/filter/input.php @@ -67,6 +67,14 @@ class JFilterInput */ public $xssAuto; + /** + * A flag for Unicode Supplementary Characters (4-byte Unicode character) stripping. + * + * @var integer + * @since CMS 3.5.0 + */ + public $stripUSC = 0; + /** * The list of the default blacklisted tags. * @@ -120,10 +128,11 @@ class JFilterInput * @param integer $tagsMethod WhiteList method = 0, BlackList method = 1 * @param integer $attrMethod WhiteList method = 0, BlackList method = 1 * @param integer $xssAuto Only auto clean essentials = 0, Allow clean blacklisted tags/attr = 1 + * @param integer $stripUSC Strip 4-byte unicode characters = 1, no strip = 0, ask the database driver = -1 * * @since 11.1 */ - public function __construct($tagsArray = array(), $attrArray = array(), $tagsMethod = 0, $attrMethod = 0, $xssAuto = 1) + public function __construct($tagsArray = array(), $attrArray = array(), $tagsMethod = 0, $attrMethod = 0, $xssAuto = 1, $stripUSC = -1) { // Make sure user defined arrays are in lowercase $tagsArray = array_map('strtolower', (array) $tagsArray); @@ -135,6 +144,31 @@ public function __construct($tagsArray = array(), $attrArray = array(), $tagsMet $this->tagsMethod = $tagsMethod; $this->attrMethod = $attrMethod; $this->xssAuto = $xssAuto; + $this->stripUSC = $stripUSC; + + /** + * If Unicode Supplementary Characters stripping is not set we have to check with the database driver. If the + * driver does not support USCs (i.e. there is no utf8mb4 support) we will enable USC stripping. + */ + if ($this->stripUSC == -1) + { + try + { + // Get the database driver + $db = JFactory::getDbo(); + + // This trick is required to let the driver determine the utf-8 multibyte support + $db->connect(); + + // And now we can decide if we should strip USCs + $this->stripUSC = $db->hasUTF8mb4Support() ? 0 : 1; + } + catch (Exception $e) + { + // Could not connect to MySQL. Strip USC to be on the safe side. + $this->stripUSC = 1; + } + } } /** @@ -145,18 +179,19 @@ public function __construct($tagsArray = array(), $attrArray = array(), $tagsMet * @param integer $tagsMethod WhiteList method = 0, BlackList method = 1 * @param integer $attrMethod WhiteList method = 0, BlackList method = 1 * @param integer $xssAuto Only auto clean essentials = 0, Allow clean blacklisted tags/attr = 1 + * @param integer $stripUSC Strip 4-byte unicode characters = 1, no strip = 0, ask the database driver = -1 * * @return JFilterInput The JFilterInput object. * * @since 11.1 */ - public static function &getInstance($tagsArray = array(), $attrArray = array(), $tagsMethod = 0, $attrMethod = 0, $xssAuto = 1) + public static function &getInstance($tagsArray = array(), $attrArray = array(), $tagsMethod = 0, $attrMethod = 0, $xssAuto = 1, $stripUSC = -1) { $sig = md5(serialize(array($tagsArray, $attrArray, $tagsMethod, $attrMethod, $xssAuto))); if (empty(self::$instances[$sig])) { - self::$instances[$sig] = new JFilterInput($tagsArray, $attrArray, $tagsMethod, $attrMethod, $xssAuto); + self::$instances[$sig] = new JFilterInput($tagsArray, $attrArray, $tagsMethod, $attrMethod, $xssAuto, $stripUSC); } return self::$instances[$sig]; @@ -192,6 +227,13 @@ public static function &getInstance($tagsArray = array(), $attrArray = array(), */ public function clean($source, $type = 'string') { + // Strip Unicode Supplementary Characters when requested to do so + if ($this->stripUSC) + { + // Alternatively: preg_replace('/[\x{10000}-\x{10FFFF}]/u', "\xE2\xAF\x91", $source) but it'd be slower. + $source = $this->stripUSC($source); + } + // Handle the type constraint switch (strtoupper($type)) { @@ -1086,4 +1128,33 @@ protected function _stripCSSExpressions($source) return $return; } + + /** + * Recursively strip Unicode Supplementary Characters from the source. Not: objects cannot be filtered. + * + * @param mixed $source The data to filter + * + * @return mixed The filtered result + */ + protected function stripUSC($source) + { + if (is_object($source)) + { + return $source; + } + + if (is_array($source)) + { + $filteredArray = array(); + + foreach ($source as $k => $v) + { + $filteredArray[$k] = $this->stripUSC($v); + } + + return $filteredArray; + } + + return preg_replace('/[\xF0-\xF7].../s', "\xE2\xAF\x91", $source); + } } diff --git a/tests/unit/core/mock/database/driver.php b/tests/unit/core/mock/database/driver.php index 1baa03cb01fe8..35c93d7462db1 100644 --- a/tests/unit/core/mock/database/driver.php +++ b/tests/unit/core/mock/database/driver.php @@ -51,6 +51,7 @@ public static function create($test, $driver = '', array $extraMethods = array() 'freeResult', 'getAffectedRows', 'getCollation', + 'getConnectionCollation', 'getConnectors', 'getDateFormat', 'getErrorMsg', diff --git a/tests/unit/suites/libraries/joomla/database/stubs/nosqldriver.php b/tests/unit/suites/libraries/joomla/database/stubs/nosqldriver.php index 06a337fde10cb..c6b0254ba5e73 100644 --- a/tests/unit/suites/libraries/joomla/database/stubs/nosqldriver.php +++ b/tests/unit/suites/libraries/joomla/database/stubs/nosqldriver.php @@ -199,6 +199,17 @@ public function getCollation() return false; } + /** + * Method to get the database connection collation, as reported by the driver. If the connector doesn't support + * reporting this value please return an empty string. + * + * @return string + */ + public function getConnectionCollation() + { + return false; + } + /** * Get the number of returned rows for the previous executed SQL statement. *