recaptime.dev / phorge
@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.) hq.recaptime.dev/wiki/Phorge
phorge phabricator
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

Fix various MySQL version issues with new charset stuff

Summary:
Ref T1191. Notable stuff:

- Adds `--disable-utf8mb4` to `bin/storage` to make it easier to test what things will (approximately) do on old MySQL. This isn't 100% perfect but should catch all the major stuff. It basically makes us pretend the server is an old server.
- Require utf8mb4 to dump a quickstart.
- Fix some issues with quickstart generation, notably special casing the FULLTEXT handling.
- Add an `--unsafe` flag to `bin/storage adjust` to let it truncate data to fix schemata.
- Fix some old patches which don't work if the default table charset is utf8mb4.

Test Plan:
- Dumped a quickstart.
- Loaded the quickstart with utf8mb4.
- Loaded the quickstart with `--disable-utf8mb4` (verified that we get binary columns, etc).
- Adjusted schema with `--disable-utf8mb4` (got a long adjustment with binary columns, some truncation stuff with weird edge case test data).
- Adjusted schema with `--disable-utf8mb4 --unsafe` (got truncations and clean adjust).
- Adjusted schema back without `--disable-utf8mb4` (got a long adjustment with utf8mb4 columns, some invalid data on truncated utf8).
- Adjusted schema without `--disable-utf8mb4`, but with `--unsafe` (got truncations on the invalid data).

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley

Maniphest Tasks: T1191

Differential Revision: https://secure.phabricator.com/D10757

epriestley 917da084 bf66c908

+153 -59
+26 -26
resources/celerity/map.php
··· 855 855 'javelin-workflow', 856 856 'phabricator-draggable-list', 857 857 ), 858 - '7319e029' => array( 859 - 'javelin-behavior', 860 - 'javelin-dom', 861 - ), 862 858 '06e05112' => array( 863 859 'javelin-behavior', 864 860 'javelin-stratcom', ··· 1276 1272 'phabricator-phtize', 1277 1273 'changeset-view-manager', 1278 1274 ), 1275 + '7319e029' => array( 1276 + 'javelin-behavior', 1277 + 'javelin-dom', 1278 + ), 1279 1279 '76b9fc3e' => array( 1280 1280 'javelin-behavior', 1281 1281 'javelin-stratcom', ··· 1335 1335 'javelin-uri', 1336 1336 'javelin-behavior-device', 1337 1337 'phabricator-title', 1338 - ), 1339 - 42126667 => array( 1340 - 'javelin-behavior', 1341 - 'javelin-dom', 1342 - 'javelin-request', 1343 - ), 1344 - 48086888 => array( 1345 - 'javelin-behavior', 1346 - 'javelin-dom', 1347 - 'javelin-workflow', 1348 - ), 1349 - 60479091 => array( 1350 - 'phabricator-busy', 1351 - 'javelin-behavior', 1352 - ), 1353 - 82439934 => array( 1354 - 'javelin-behavior', 1355 - 'javelin-dom', 1356 - 'javelin-util', 1357 - 'javelin-stratcom', 1358 - 'javelin-workflow', 1359 - 'phabricator-draggable-list', 1360 1338 ), 1361 1339 '7e41274a' => array( 1362 1340 'javelin-install', ··· 1961 1939 'javelin-util', 1962 1940 'phabricator-prefab', 1963 1941 'javelin-json', 1942 + ), 1943 + 42126667 => array( 1944 + 'javelin-behavior', 1945 + 'javelin-dom', 1946 + 'javelin-request', 1947 + ), 1948 + 48086888 => array( 1949 + 'javelin-behavior', 1950 + 'javelin-dom', 1951 + 'javelin-workflow', 1952 + ), 1953 + 60479091 => array( 1954 + 'phabricator-busy', 1955 + 'javelin-behavior', 1956 + ), 1957 + 82439934 => array( 1958 + 'javelin-behavior', 1959 + 'javelin-dom', 1960 + 'javelin-util', 1961 + 'javelin-stratcom', 1962 + 'javelin-workflow', 1963 + 'phabricator-draggable-list', 1964 1964 ), 1965 1965 ), 1966 1966 'packages' => array(
+1 -1
resources/sql/patches/000.project.sql
··· 1 1 create table {$NAMESPACE}_project.project ( 2 2 id int unsigned not null auto_increment primary key, 3 - name varchar(255) not null, 3 + name varchar(255) COLLATE `binary` not null, 4 4 unique key (name), 5 5 phid varchar(64) binary not null, 6 6 authorPHID varchar(64) binary not null,
+2 -2
resources/sql/patches/002.oauth.sql
··· 1 1 create table {$NAMESPACE}_user.user_oauthinfo ( 2 2 id int unsigned not null auto_increment primary key, 3 3 userID int unsigned not null, 4 - oauthProvider varchar(255) not null, 5 - oauthUID varchar(255) not null, 4 + oauthProvider varchar(255) COLLATE `binary` not null, 5 + oauthUID varchar(255) COLLATE `binary` not null, 6 6 unique key (userID, oauthProvider), 7 7 unique key (oauthProvider, oauthUID), 8 8 dateCreated int unsigned not null,
+1 -1
resources/sql/patches/004.daemonrepos.sql
··· 23 23 ); 24 24 25 25 create table {$NAMESPACE}_timeline.timeline_cursor ( 26 - name varchar(255) not null primary key, 26 + name varchar(255) COLLATE `binary` not null primary key, 27 27 position int unsigned not null 28 28 );
+1 -1
resources/sql/patches/010.herald.sql
··· 7 7 8 8 CREATE TABLE {$NAMESPACE}_herald.herald_rule ( 9 9 id int unsigned not null auto_increment primary key, 10 - name varchar(255) not null, 10 + name varchar(255) COLLATE `binary` not null, 11 11 authorPHID varchar(64) binary not null, 12 12 contentType varchar(255) not null, 13 13 mustMatchAll bool not null,
+1 -1
resources/sql/patches/011.badcommit.sql
··· 1 1 CREATE TABLE {$NAMESPACE}_repository.repository_badcommit ( 2 - fullCommitName varchar(255) binary not null primary key, 2 + fullCommitName varchar(255) COLLATE `binary` not null primary key, 3 3 description longblob not null 4 4 );
+1 -1
resources/sql/patches/018.owners.sql
··· 2 2 id int unsigned not null auto_increment primary key, 3 3 phid varchar(64) binary not null, 4 4 unique key(phid), 5 - name varchar(255) not null, 5 + name varchar(255) COLLATE `binary` not null, 6 6 unique key(name), 7 7 description text not null, 8 8 primaryOwnerPHID varchar(64) binary
+1 -1
resources/sql/patches/019.arcprojects.sql
··· 2 2 id int unsigned not null auto_increment primary key, 3 3 phid varchar(64) binary not null, 4 4 unique key(phid), 5 - name varchar(255) not null, 5 + name varchar(255) COLLATE `binary` not null, 6 6 unique key (name), 7 7 repositoryID int unsigned 8 8 );
+1 -1
resources/sql/patches/035.proxyimage.sql
··· 1 1 CREATE TABLE {$NAMESPACE}_file.file_proxyimage ( 2 2 id int unsigned not null primary key auto_increment, 3 - uri varchar(255) binary not null, 3 + uri varchar(255) COLLATE `binary` not null, 4 4 unique key(uri), 5 5 filePHID varchar(64) binary not null 6 6 ) ENGINE=InnoDB;
+2 -2
resources/sql/patches/040.transform.sql
··· 1 1 CREATE TABLE {$NAMESPACE}_file.file_transformedfile ( 2 2 id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, 3 - originalPHID varchar(64) BINARY NOT NULL, 4 - transform varchar(255) BINARY NOT NULL, 3 + originalPHID varchar(64) COLLATE `binary` NOT NULL, 4 + transform varchar(255) COLLATE `binary` NOT NULL, 5 5 unique key (originalPHID, transform), 6 6 transformedPHID varchar(64) BINARY NOT NULL, 7 7 key (transformedPHID),
+1 -1
resources/sql/patches/068.maniphestauxiliarystorage.sql
··· 1 1 create table {$NAMESPACE}_maniphest.maniphest_taskauxiliarystorage 2 2 (id int unsigned not null auto_increment primary key, 3 3 taskPHID varchar(64) binary not null, 4 - name varchar(255) not null, 4 + name varchar(255) COLLATE `binary` not null, 5 5 value varchar(255) not null, 6 6 unique key (taskPHID,name), 7 7 dateCreated int unsigned not null,
+8
scripts/sql/manage_storage.php
··· 64 64 'help' => 'Do not actually change anything, just show what would be '. 65 65 'changed.', 66 66 ), 67 + array( 68 + 'name' => 'disable-utf8mb4', 69 + 'help' => pht( 70 + 'Disable utf8mb4, even if the database supports it. This is an '. 71 + 'advanced feature used for testing changes to Phabricator; you '. 72 + 'should not normally use this flag.'), 73 + ) 67 74 )); 68 75 } catch (PhutilArgumentUsageException $ex) { 69 76 $args->printUsageException($ex); ··· 126 133 $api->setPort($default_port); 127 134 $api->setPassword($password); 128 135 $api->setNamespace($args->getArg('namespace')); 136 + $api->setDisableUTF8MB4($args->getArg('disable-utf8mb4')); 129 137 130 138 try { 131 139 queryfx(
+7 -8
src/applications/config/schema/PhabricatorConfigSchemaQuery.php
··· 153 153 154 154 public function loadExpectedSchema() { 155 155 $databases = $this->getDatabaseNames(); 156 - 157 - $api = $this->getAPI(); 158 - 159 - $charset_info = $api->getCharsetInfo(); 160 - list($charset, $collate_text, $collate_sort) = $charset_info; 156 + $info = $this->getAPI()->getCharsetInfo(); 161 157 162 158 $specs = id(new PhutilSymbolLoader()) 163 159 ->setAncestorClass('PhabricatorConfigSchemaSpec') ··· 166 162 $server_schema = new PhabricatorConfigServerSchema(); 167 163 foreach ($specs as $spec) { 168 164 $spec 169 - ->setUTF8Charset($charset) 170 - ->setUTF8BinaryCollation($collate_text) 171 - ->setUTF8SortingCollation($collate_sort) 165 + ->setUTF8Charset( 166 + $info[PhabricatorStorageManagementAPI::CHARSET_DEFAULT]) 167 + ->setUTF8BinaryCollation( 168 + $info[PhabricatorStorageManagementAPI::COLLATE_TEXT]) 169 + ->setUTF8SortingCollation( 170 + $info[PhabricatorStorageManagementAPI::COLLATE_SORT]) 172 171 ->setServer($server_schema) 173 172 ->buildSchemata($server_schema); 174 173 }
+1 -1
src/applications/people/storage/PhabricatorUserSchemaSpec.php
··· 9 9 id(new PhabricatorUser())->getApplicationName(), 10 10 PhabricatorUser::NAMETOKEN_TABLE, 11 11 array( 12 - 'token' => 'text255', 12 + 'token' => 'sort255', 13 13 'userID' => 'id', 14 14 ), 15 15 array(
+46 -9
src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
··· 8 8 private $password; 9 9 private $namespace; 10 10 private $conns = array(); 11 + private $disableUTF8MB4; 12 + 13 + const CHARSET_DEFAULT = 'CHARSET'; 14 + const CHARSET_FULLTEXT = 'CHARSET_FULLTEXT'; 15 + const COLLATE_TEXT = 'COLLATE_TEXT'; 16 + const COLLATE_SORT = 'COLLATE_SORT'; 17 + const COLLATE_FULLTEXT = 'COLLATE_FULLTEXT'; 18 + 19 + public function setDisableUTF8MB4($disable_utf8_mb4) { 20 + $this->disableUTF8MB4 = $disable_utf8_mb4; 21 + return $this; 22 + } 23 + 24 + public function getDisableUTF8MB4() { 25 + return $this->disableUTF8MB4; 26 + } 11 27 12 28 public function setNamespace($namespace) { 13 29 $this->namespace = $namespace; ··· 110 126 111 127 public function createDatabase($fragment) { 112 128 $info = $this->getCharsetInfo(); 113 - list($charset, $collate_text, $collate_sort) = $info; 114 129 115 130 queryfx( 116 131 $this->getConn(null), 117 132 'CREATE DATABASE IF NOT EXISTS %T COLLATE %T', 118 133 $this->getDatabaseName($fragment), 119 - $collate_text); 134 + $info[self::COLLATE_TEXT]); 120 135 } 121 136 122 137 public function createTable($fragment, $table, array $cols) { ··· 187 202 $conn = $this->getConn(null); 188 203 189 204 $charset_info = $this->getCharsetInfo(); 190 - list($charset, $collate_text, $collate_sort) = $charset_info; 205 + foreach ($charset_info as $key => $value) { 206 + $charset_info[$key] = qsprintf($conn, '%T', $value); 207 + } 191 208 192 209 foreach ($queries as $query) { 193 210 $query = str_replace('{$NAMESPACE}', $this->namespace, $query); 194 - $query = str_replace('{$CHARSET}', $charset, $query); 195 - $escaped_text = qsprintf($conn, '%T', $collate_text); 196 - $query = str_replace('{$COLLATE_TEXT}', $escaped_text, $query); 197 - $escaped_text = qsprintf($conn, '%T', $collate_sort); 198 - $query = str_replace('{$COLLATE_SORT}', $escaped_text, $query); 211 + 212 + foreach ($charset_info as $key => $value) { 213 + $query = str_replace('{$'.$key.'}', $value, $query); 214 + } 215 + 199 216 queryfx( 200 217 $conn, 201 218 '%Q', ··· 209 226 } 210 227 211 228 public function isCharacterSetAvailable($character_set) { 229 + if ($character_set == 'utf8mb4') { 230 + if ($this->getDisableUTF8MB4()) { 231 + return false; 232 + } 233 + } 234 + 212 235 $conn = $this->getConn(null); 213 236 214 237 $result = queryfx_one( ··· 226 249 // collation. This is most correct, and will sort properly. 227 250 228 251 $charset = 'utf8mb4'; 252 + $charset_full = 'utf8mb4'; 229 253 $collate_text = 'utf8mb4_bin'; 230 254 $collate_sort = 'utf8mb4_unicode_ci'; 255 + $collate_full = 'utf8mb4_unicode_ci'; 231 256 } else { 232 257 // If utf8mb4 is not available, we use binary. This allows us to store 233 258 // 4-byte unicode characters. This has some tradeoffs: ··· 238 263 // It's possible that strings will be truncated in the middle of a 239 264 // character on insert. We encourage users to set STRICT_ALL_TABLES 240 265 // to prevent this. 266 + // 267 + // There's no valid collation we can use to get a fulltext index on 268 + // 4-byte unicode characters: we can't add a fulltext key to a binary 269 + // column. 241 270 242 271 $charset = 'binary'; 272 + $charset_full = 'utf8'; 243 273 $collate_text = 'binary'; 244 274 $collate_sort = 'binary'; 275 + $collate_full = 'utf8_general_ci'; 245 276 } 246 277 247 - return array($charset, $collate_text, $collate_sort); 278 + return array( 279 + self::CHARSET_DEFAULT => $charset, 280 + self::CHARSET_FULLTEXT => $charset_full, 281 + self::COLLATE_TEXT => $collate_text, 282 + self::COLLATE_SORT => $collate_sort, 283 + self::COLLATE_FULLTEXT => $collate_full, 284 + ); 248 285 } 249 286 250 287 }
+30 -3
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php
··· 10 10 ->setSynopsis( 11 11 pht( 12 12 'Make schemata adjustments to correct issues with characters sets, '. 13 - 'collations, and keys.')); 13 + 'collations, and keys.')) 14 + ->setArguments( 15 + array( 16 + array( 17 + 'name' => 'unsafe', 18 + 'help' => pht( 19 + 'Permit adjustments which truncate data. This option may '. 20 + 'destroy some data, but the lost data is usually not '. 21 + 'important (most commonly, the ends of very long object '. 22 + 'titles).'), 23 + ), 24 + )); 14 25 } 15 26 16 27 public function execute(PhutilArgumentParser $args) { 17 28 $force = $args->getArg('force'); 29 + $unsafe = $args->getArg('unsafe'); 18 30 19 31 $this->requireAllPatchesApplied(); 20 - return $this->adjustSchemata($force); 32 + return $this->adjustSchemata($force, $unsafe); 21 33 } 22 34 23 35 private function requireAllPatchesApplied() { ··· 59 71 return array($comp, $expect, $actual); 60 72 } 61 73 62 - private function adjustSchemata($force) { 74 + private function adjustSchemata($force, $unsafe) { 63 75 $console = PhutilConsole::getConsole(); 64 76 65 77 $console->writeOut( ··· 137 149 138 150 $api = $this->getAPI(); 139 151 $conn = $api->getConn(null); 152 + 153 + if ($unsafe) { 154 + queryfx($conn, 'SET SESSION sql_mode = %s', ''); 155 + } else { 156 + queryfx($conn, 'SET SESSION sql_mode = %s', 'STRICT_ALL_TABLES'); 157 + } 140 158 141 159 $failed = array(); 142 160 ··· 326 344 $console->writeOut( 327 345 "\n%s\n", 328 346 pht('Failed to make some schema adjustments, detailed above.')); 347 + 348 + if (!$unsafe) { 349 + $console->writeOut( 350 + "%s\n", 351 + pht( 352 + 'Migrations which fail with certain types of errors (including '. 353 + '"#1406 Data Too Long" and "#1366 Incorrect String Value") can be '. 354 + 'forced to complete by running again with `--unsafe`.')); 355 + } 329 356 330 357 return 1; 331 358 }
+23
src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php
··· 33 33 34 34 $bin = dirname(phutil_get_library_root('phabricator')).'/bin/storage'; 35 35 36 + if (!$this->getAPI()->isCharacterSetAvailable('utf8mb4')) { 37 + throw new PhutilArgumentUsageException( 38 + pht( 39 + 'You can only generate a new quickstart file if MySQL supports '. 40 + 'the utf8mb4 character set (available in MySQL 5.5 and newer). The '. 41 + 'configured server does not support utf8mb4.')); 42 + } 43 + 36 44 $err = phutil_passthru( 37 45 '%s upgrade --force --no-quickstart --namespace %s', 38 46 $bin, ··· 73 81 $namespace, 74 82 '{$NAMESPACE}', 75 83 $dump); 84 + 85 + // NOTE: This is a hack. We can not use `binary` for this column, because 86 + // it is part of a fulltext index. 87 + $old = $dump; 88 + $dump = preg_replace( 89 + '/`corpus` longtext CHARACTER SET .* COLLATE .*,/mi', 90 + '`corpus` longtext CHARACTER SET {$CHARSET_FULLTEXT} '. 91 + 'COLLATE {$COLLATE_FULLTEXT},', 92 + $dump); 93 + if ($dump == $old) { 94 + // If we didn't make any changes, yell about it. We'll produce an invalid 95 + // dump otherwise. 96 + throw new PhutilArgumentUsageException( 97 + pht('Failed to apply hack to adjust FULLTEXT search column!')); 98 + } 76 99 77 100 $dump = str_replace( 78 101 'utf8mb4_bin',