Apply storage adjustments as part of storage upgrade

+2 -603

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php

··· 27 27 public function execute(PhutilArgumentParser $args) { 28 28 $force = $args->getArg('force'); 29 29 $unsafe = $args->getArg('unsafe'); 30 + $dry_run = $args->getArg('dryrun'); 30 31 31 32 $this->requireAllPatchesApplied(); 32 - return $this->adjustSchemata($force, $unsafe); 33 + return $this->adjustSchemata($force, $unsafe, $dry_run); 33 34 } 34 35 35 36 private function requireAllPatchesApplied() { ··· 58 59 'Run `storage status` to show patch status, and `storage upgrade` '. 59 60 'to apply missing patches.')); 60 61 } 61 - } 62 - 63 - private function loadSchemata() { 64 - $query = id(new PhabricatorConfigSchemaQuery()) 65 - ->setAPI($this->getAPI()); 66 - 67 - $actual = $query->loadActualSchema(); 68 - $expect = $query->loadExpectedSchema(); 69 - $comp = $query->buildComparisonSchema($expect, $actual); 70 - 71 - return array($comp, $expect, $actual); 72 - } 73 - 74 - private function adjustSchemata($force, $unsafe) { 75 - $console = PhutilConsole::getConsole(); 76 - 77 - $console->writeOut( 78 - "%s\n", 79 - pht('Verifying database schemata...')); 80 - 81 - list($adjustments, $errors) = $this->findAdjustments(); 82 - $api = $this->getAPI(); 83 - 84 - if (!$adjustments) { 85 - $console->writeOut( 86 - "%s\n", 87 - pht('Found no adjustments for schemata.')); 88 - 89 - return $this->printErrors($errors, 0); 90 - } 91 - 92 - if (!$force && !$api->isCharacterSetAvailable('utf8mb4')) { 93 - $message = pht( 94 - "You have an old version of MySQL (older than 5.5) which does not ". 95 - "support the utf8mb4 character set. If you apply adjustments now ". 96 - "and later update MySQL to 5.5 or newer, you'll need to apply ". 97 - "adjustments again (and they will take a long time).\n\n". 98 - "You can exit this workflow, update MySQL now, and then run this ". 99 - "workflow again. This is recommended, but may cause a lot of downtime ". 100 - "right now.\n\n". 101 - "You can exit this workflow, continue using Phabricator without ". 102 - "applying adjustments, update MySQL at a later date, and then run ". 103 - "this workflow again. This is also a good approach, and will let you ". 104 - "delay downtime until later.\n\n". 105 - "You can proceed with this workflow, and then optionally update ". 106 - "MySQL at a later date. After you do, you'll need to apply ". 107 - "adjustments again.\n\n". 108 - "For more information, see \"Managing Storage Adjustments\" in ". 109 - "the documentation."); 110 - 111 - $console->writeOut( 112 - "\n**<bg:yellow> %s </bg>**\n\n%s\n", 113 - pht('OLD MySQL VERSION'), 114 - phutil_console_wrap($message)); 115 - 116 - $prompt = pht('Continue with old MySQL version?'); 117 - if (!phutil_console_confirm($prompt, $default_no = true)) { 118 - return; 119 - } 120 - } 121 - 122 - $table = id(new PhutilConsoleTable()) 123 - ->addColumn('database', array('title' => pht('Database'))) 124 - ->addColumn('table', array('title' => pht('Table'))) 125 - ->addColumn('name', array('title' => pht('Name'))) 126 - ->addColumn('info', array('title' => pht('Issues'))); 127 - 128 - foreach ($adjustments as $adjust) { 129 - $info = array(); 130 - foreach ($adjust['issues'] as $issue) { 131 - $info[] = PhabricatorConfigStorageSchema::getIssueName($issue); 132 - } 133 - 134 - $table->addRow(array( 135 - 'database' => $adjust['database'], 136 - 'table' => idx($adjust, 'table'), 137 - 'name' => idx($adjust, 'name'), 138 - 'info' => implode(', ', $info), 139 - )); 140 - } 141 - 142 - $console->writeOut("\n\n"); 143 - 144 - $table->draw(); 145 - 146 - if (!$force) { 147 - $console->writeOut( 148 - "\n%s\n", 149 - pht( 150 - "Found %s issues(s) with schemata, detailed above.\n\n". 151 - "You can review issues in more detail from the web interface, ". 152 - "in Config > Database Status. To better understand the adjustment ". 153 - "workflow, see \"Managing Storage Adjustments\" in the ". 154 - "documentation.\n\n". 155 - "MySQL needs to copy table data to make some adjustments, so these ". 156 - "migrations may take some time.", 157 - new PhutilNumber(count($adjustments)))); 158 - 159 - $prompt = pht('Fix these schema issues?'); 160 - if (!phutil_console_confirm($prompt, $default_no = true)) { 161 - return; 162 - } 163 - } 164 - 165 - $console->writeOut( 166 - "%s\n", 167 - pht('Dropping caches, for faster migrations...')); 168 - 169 - $root = dirname(phutil_get_library_root('phabricator')); 170 - $bin = $root.'/bin/cache'; 171 - phutil_passthru('%s purge --purge-all', $bin); 172 - 173 - $console->writeOut( 174 - "%s\n", 175 - pht('Fixing schema issues...')); 176 - 177 - $conn = $api->getConn(null); 178 - 179 - if ($unsafe) { 180 - queryfx($conn, 'SET SESSION sql_mode = %s', ''); 181 - } else { 182 - queryfx($conn, 'SET SESSION sql_mode = %s', 'STRICT_ALL_TABLES'); 183 - } 184 - 185 - $failed = array(); 186 - 187 - // We make changes in several phases. 188 - $phases = array( 189 - // Drop surplus autoincrements. This allows us to drop primary keys on 190 - // autoincrement columns. 191 - 'drop_auto', 192 - 193 - // Drop all keys we're going to adjust. This prevents them from 194 - // interfering with column changes. 195 - 'drop_keys', 196 - 197 - // Apply all database, table, and column changes. 198 - 'main', 199 - 200 - // Restore adjusted keys. 201 - 'add_keys', 202 - 203 - // Add missing autoincrements. 204 - 'add_auto', 205 - ); 206 - 207 - $bar = id(new PhutilConsoleProgressBar()) 208 - ->setTotal(count($adjustments) * count($phases)); 209 - 210 - foreach ($phases as $phase) { 211 - foreach ($adjustments as $adjust) { 212 - try { 213 - switch ($adjust['kind']) { 214 - case 'database': 215 - if ($phase == 'main') { 216 - queryfx( 217 - $conn, 218 - 'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s', 219 - $adjust['database'], 220 - $adjust['charset'], 221 - $adjust['collation']); 222 - } 223 - break; 224 - case 'table': 225 - if ($phase == 'main') { 226 - queryfx( 227 - $conn, 228 - 'ALTER TABLE %T.%T COLLATE = %s', 229 - $adjust['database'], 230 - $adjust['table'], 231 - $adjust['collation']); 232 - } 233 - break; 234 - case 'column': 235 - $apply = false; 236 - $auto = false; 237 - $new_auto = idx($adjust, 'auto'); 238 - if ($phase == 'drop_auto') { 239 - if ($new_auto === false) { 240 - $apply = true; 241 - $auto = false; 242 - } 243 - } else if ($phase == 'main') { 244 - $apply = true; 245 - if ($new_auto === false) { 246 - $auto = false; 247 - } else { 248 - $auto = $adjust['is_auto']; 249 - } 250 - } else if ($phase == 'add_auto') { 251 - if ($new_auto === true) { 252 - $apply = true; 253 - $auto = true; 254 - } 255 - } 256 - 257 - if ($apply) { 258 - $parts = array(); 259 - 260 - if ($auto) { 261 - $parts[] = qsprintf( 262 - $conn, 263 - 'AUTO_INCREMENT'); 264 - } 265 - 266 - if ($adjust['charset']) { 267 - $parts[] = qsprintf( 268 - $conn, 269 - 'CHARACTER SET %Q COLLATE %Q', 270 - $adjust['charset'], 271 - $adjust['collation']); 272 - } 273 - 274 - queryfx( 275 - $conn, 276 - 'ALTER TABLE %T.%T MODIFY %T %Q %Q %Q', 277 - $adjust['database'], 278 - $adjust['table'], 279 - $adjust['name'], 280 - $adjust['type'], 281 - implode(' ', $parts), 282 - $adjust['nullable'] ? 'NULL' : 'NOT NULL'); 283 - } 284 - break; 285 - case 'key': 286 - if (($phase == 'drop_keys') && $adjust['exists']) { 287 - if ($adjust['name'] == 'PRIMARY') { 288 - $key_name = 'PRIMARY KEY'; 289 - } else { 290 - $key_name = qsprintf($conn, 'KEY %T', $adjust['name']); 291 - } 292 - 293 - queryfx( 294 - $conn, 295 - 'ALTER TABLE %T.%T DROP %Q', 296 - $adjust['database'], 297 - $adjust['table'], 298 - $key_name); 299 - } 300 - 301 - if (($phase == 'add_keys') && $adjust['keep']) { 302 - // Different keys need different creation syntax. Notable 303 - // special cases are primary keys and fulltext keys. 304 - if ($adjust['name'] == 'PRIMARY') { 305 - $key_name = 'PRIMARY KEY'; 306 - } else if ($adjust['indexType'] == 'FULLTEXT') { 307 - $key_name = qsprintf($conn, 'FULLTEXT %T', $adjust['name']); 308 - } else { 309 - if ($adjust['unique']) { 310 - $key_name = qsprintf( 311 - $conn, 312 - 'UNIQUE KEY %T', 313 - $adjust['name']); 314 - } else { 315 - $key_name = qsprintf( 316 - $conn, 317 - '/* NONUNIQUE */ KEY %T', 318 - $adjust['name']); 319 - } 320 - } 321 - 322 - queryfx( 323 - $conn, 324 - 'ALTER TABLE %T.%T ADD %Q (%Q)', 325 - $adjust['database'], 326 - $adjust['table'], 327 - $key_name, 328 - implode(', ', $adjust['columns'])); 329 - } 330 - break; 331 - default: 332 - throw new Exception( 333 - pht('Unknown schema adjustment kind "%s"!', $adjust['kind'])); 334 - } 335 - } catch (AphrontQueryException $ex) { 336 - $failed[] = array($adjust, $ex); 337 - } 338 - $bar->update(1); 339 - } 340 - } 341 - $bar->done(); 342 - 343 - if (!$failed) { 344 - $console->writeOut( 345 - "%s\n", 346 - pht('Completed fixing all schema issues.')); 347 - 348 - $err = 0; 349 - } else { 350 - $table = id(new PhutilConsoleTable()) 351 - ->addColumn('target', array('title' => pht('Target'))) 352 - ->addColumn('error', array('title' => pht('Error'))); 353 - 354 - foreach ($failed as $failure) { 355 - list($adjust, $ex) = $failure; 356 - 357 - $pieces = array_select_keys( 358 - $adjust, 359 - array('database', 'table', 'name')); 360 - $pieces = array_filter($pieces); 361 - $target = implode('.', $pieces); 362 - 363 - $table->addRow( 364 - array( 365 - 'target' => $target, 366 - 'error' => $ex->getMessage(), 367 - )); 368 - } 369 - 370 - $console->writeOut("\n"); 371 - $table->draw(); 372 - $console->writeOut( 373 - "\n%s\n", 374 - pht('Failed to make some schema adjustments, detailed above.')); 375 - $console->writeOut( 376 - "%s\n", 377 - pht( 378 - 'For help troubleshooting adjustments, see "Managing Storage '. 379 - 'Adjustments" in the documentation.')); 380 - 381 - $err = 1; 382 - } 383 - 384 - return $this->printErrors($errors, $err); 385 - } 386 - 387 - private function findAdjustments() { 388 - list($comp, $expect, $actual) = $this->loadSchemata(); 389 - 390 - $issue_charset = PhabricatorConfigStorageSchema::ISSUE_CHARSET; 391 - $issue_collation = PhabricatorConfigStorageSchema::ISSUE_COLLATION; 392 - $issue_columntype = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE; 393 - $issue_surpluskey = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY; 394 - $issue_missingkey = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY; 395 - $issue_columns = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS; 396 - $issue_unique = PhabricatorConfigStorageSchema::ISSUE_UNIQUE; 397 - $issue_longkey = PhabricatorConfigStorageSchema::ISSUE_LONGKEY; 398 - $issue_auto = PhabricatorConfigStorageSchema::ISSUE_AUTOINCREMENT; 399 - 400 - $adjustments = array(); 401 - $errors = array(); 402 - foreach ($comp->getDatabases() as $database_name => $database) { 403 - foreach ($this->findErrors($database) as $issue) { 404 - $errors[] = array( 405 - 'database' => $database_name, 406 - 'issue' => $issue, 407 - ); 408 - } 409 - 410 - $expect_database = $expect->getDatabase($database_name); 411 - $actual_database = $actual->getDatabase($database_name); 412 - 413 - if (!$expect_database || !$actual_database) { 414 - // If there's a real issue here, skip this stuff. 415 - continue; 416 - } 417 - 418 - $issues = array(); 419 - if ($database->hasIssue($issue_charset)) { 420 - $issues[] = $issue_charset; 421 - } 422 - if ($database->hasIssue($issue_collation)) { 423 - $issues[] = $issue_collation; 424 - } 425 - 426 - if ($issues) { 427 - $adjustments[] = array( 428 - 'kind' => 'database', 429 - 'database' => $database_name, 430 - 'issues' => $issues, 431 - 'charset' => $expect_database->getCharacterSet(), 432 - 'collation' => $expect_database->getCollation(), 433 - ); 434 - } 435 - 436 - foreach ($database->getTables() as $table_name => $table) { 437 - foreach ($this->findErrors($table) as $issue) { 438 - $errors[] = array( 439 - 'database' => $database_name, 440 - 'table' => $table_name, 441 - 'issue' => $issue, 442 - ); 443 - } 444 - 445 - $expect_table = $expect_database->getTable($table_name); 446 - $actual_table = $actual_database->getTable($table_name); 447 - 448 - if (!$expect_table || !$actual_table) { 449 - continue; 450 - } 451 - 452 - $issues = array(); 453 - if ($table->hasIssue($issue_collation)) { 454 - $issues[] = $issue_collation; 455 - } 456 - 457 - if ($issues) { 458 - $adjustments[] = array( 459 - 'kind' => 'table', 460 - 'database' => $database_name, 461 - 'table' => $table_name, 462 - 'issues' => $issues, 463 - 'collation' => $expect_table->getCollation(), 464 - ); 465 - } 466 - 467 - foreach ($table->getColumns() as $column_name => $column) { 468 - foreach ($this->findErrors($column) as $issue) { 469 - $errors[] = array( 470 - 'database' => $database_name, 471 - 'table' => $table_name, 472 - 'name' => $column_name, 473 - 'issue' => $issue, 474 - ); 475 - } 476 - 477 - $expect_column = $expect_table->getColumn($column_name); 478 - $actual_column = $actual_table->getColumn($column_name); 479 - 480 - if (!$expect_column || !$actual_column) { 481 - continue; 482 - } 483 - 484 - $issues = array(); 485 - if ($column->hasIssue($issue_collation)) { 486 - $issues[] = $issue_collation; 487 - } 488 - if ($column->hasIssue($issue_charset)) { 489 - $issues[] = $issue_charset; 490 - } 491 - if ($column->hasIssue($issue_columntype)) { 492 - $issues[] = $issue_columntype; 493 - } 494 - if ($column->hasIssue($issue_auto)) { 495 - $issues[] = $issue_auto; 496 - } 497 - 498 - if ($issues) { 499 - if ($expect_column->getCharacterSet() === null) { 500 - // For non-text columns, we won't be specifying a collation or 501 - // character set. 502 - $charset = null; 503 - $collation = null; 504 - } else { 505 - $charset = $expect_column->getCharacterSet(); 506 - $collation = $expect_column->getCollation(); 507 - } 508 - 509 - $adjustment = array( 510 - 'kind' => 'column', 511 - 'database' => $database_name, 512 - 'table' => $table_name, 513 - 'name' => $column_name, 514 - 'issues' => $issues, 515 - 'collation' => $collation, 516 - 'charset' => $charset, 517 - 'type' => $expect_column->getColumnType(), 518 - 519 - // NOTE: We don't adjust column nullability because it is 520 - // dangerous, so always use the current nullability. 521 - 'nullable' => $actual_column->getNullable(), 522 - 523 - // NOTE: This always stores the current value, because we have 524 - // to make these updates separately. 525 - 'is_auto' => $actual_column->getAutoIncrement(), 526 - ); 527 - 528 - if ($column->hasIssue($issue_auto)) { 529 - $adjustment['auto'] = $expect_column->getAutoIncrement(); 530 - } 531 - 532 - $adjustments[] = $adjustment; 533 - } 534 - } 535 - 536 - foreach ($table->getKeys() as $key_name => $key) { 537 - foreach ($this->findErrors($key) as $issue) { 538 - $errors[] = array( 539 - 'database' => $database_name, 540 - 'table' => $table_name, 541 - 'name' => $key_name, 542 - 'issue' => $issue, 543 - ); 544 - } 545 - 546 - $expect_key = $expect_table->getKey($key_name); 547 - $actual_key = $actual_table->getKey($key_name); 548 - 549 - $issues = array(); 550 - $keep_key = true; 551 - if ($key->hasIssue($issue_surpluskey)) { 552 - $issues[] = $issue_surpluskey; 553 - $keep_key = false; 554 - } 555 - 556 - if ($key->hasIssue($issue_missingkey)) { 557 - $issues[] = $issue_missingkey; 558 - } 559 - 560 - if ($key->hasIssue($issue_columns)) { 561 - $issues[] = $issue_columns; 562 - } 563 - 564 - if ($key->hasIssue($issue_unique)) { 565 - $issues[] = $issue_unique; 566 - } 567 - 568 - // NOTE: We can't really fix this, per se, but we may need to remove 569 - // the key to change the column type. In the best case, the new 570 - // column type won't be overlong and recreating the key really will 571 - // fix the issue. In the worst case, we get the right column type and 572 - // lose the key, which is still better than retaining the key having 573 - // the wrong column type. 574 - if ($key->hasIssue($issue_longkey)) { 575 - $issues[] = $issue_longkey; 576 - } 577 - 578 - if ($issues) { 579 - $adjustment = array( 580 - 'kind' => 'key', 581 - 'database' => $database_name, 582 - 'table' => $table_name, 583 - 'name' => $key_name, 584 - 'issues' => $issues, 585 - 'exists' => (bool)$actual_key, 586 - 'keep' => $keep_key, 587 - ); 588 - 589 - if ($keep_key) { 590 - $adjustment += array( 591 - 'columns' => $expect_key->getColumnNames(), 592 - 'unique' => $expect_key->getUnique(), 593 - 'indexType' => $expect_key->getIndexType(), 594 - ); 595 - } 596 - 597 - $adjustments[] = $adjustment; 598 - } 599 - } 600 - } 601 - } 602 - 603 - return array($adjustments, $errors); 604 - } 605 - 606 - private function findErrors(PhabricatorConfigStorageSchema $schema) { 607 - $result = array(); 608 - foreach ($schema->getLocalIssues() as $issue) { 609 - $status = PhabricatorConfigStorageSchema::getIssueStatus($issue); 610 - if ($status == PhabricatorConfigStorageSchema::STATUS_FAIL) { 611 - $result[] = $issue; 612 - } 613 - } 614 - return $result; 615 - } 616 - 617 - private function printErrors(array $errors, $default_return) { 618 - if (!$errors) { 619 - return $default_return; 620 - } 621 - 622 - $console = PhutilConsole::getConsole(); 623 - 624 - $table = id(new PhutilConsoleTable()) 625 - ->addColumn('target', array('title' => pht('Target'))) 626 - ->addColumn('error', array('title' => pht('Error'))); 627 - 628 - foreach ($errors as $error) { 629 - $pieces = array_select_keys( 630 - $error, 631 - array('database', 'table', 'name')); 632 - $pieces = array_filter($pieces); 633 - $target = implode('.', $pieces); 634 - 635 - $name = PhabricatorConfigStorageSchema::getIssueName($error['issue']); 636 - 637 - $table->addRow( 638 - array( 639 - 'target' => $target, 640 - 'error' => $name, 641 - )); 642 - } 643 - 644 - $console->writeOut("\n"); 645 - $table->draw(); 646 - $console->writeOut("\n"); 647 - 648 - $message = pht( 649 - "The schemata have serious errors (detailed above) which the adjustment ". 650 - "workflow can not fix.\n\n". 651 - "If you are not developing Phabricator itself, report this issue to ". 652 - "the upstream.\n\n". 653 - "If you are developing Phabricator, these errors usually indicate that ". 654 - "your schema specifications do not agree with the schemata your code ". 655 - "actually builds."); 656 - 657 - $console->writeOut( 658 - "**<bg:red> %s </bg>**\n\n%s\n", 659 - pht('SCHEMATA ERRORS'), 660 - phutil_console_wrap($message)); 661 - 662 - return 2; 663 62 } 664 63 665 64 }

+24 -7

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php

··· 13 13 array( 14 14 'name' => 'apply', 15 15 'param' => 'patch', 16 - 'help' => 'Apply __patch__ explicitly. This is an advanced '. 17 - 'feature for development and debugging; you should '. 18 - 'not normally use this flag.', 16 + 'help' => pht( 17 + 'Apply __patch__ explicitly. This is an advanced feature for '. 18 + 'development and debugging; you should not normally use this '. 19 + 'flag. This skips adjustment.'), 19 20 ), 20 21 array( 21 22 'name' => 'no-quickstart', 22 - 'help' => 'Build storage patch-by-patch from scatch, even if it '. 23 - 'could be loaded from the quickstart template.', 23 + 'help' => pht( 24 + 'Build storage patch-by-patch from scatch, even if it could '. 25 + 'be loaded from the quickstart template.'), 24 26 ), 25 27 array( 26 28 'name' => 'init-only', 27 - 'help' => 'Initialize storage only; do not apply patches.', 29 + 'help' => pht( 30 + 'Initialize storage only; do not apply patches or adjustments.'), 31 + ), 32 + array( 33 + 'name' => 'no-adjust', 34 + 'help' => pht( 35 + 'Do not apply storage adjustments after storage upgrades.'), 28 36 ), 29 37 )); 30 38 } ··· 59 67 60 68 $no_quickstart = $args->getArg('no-quickstart'); 61 69 $init_only = $args->getArg('init-only'); 70 + $no_adjust = $args->getArg('no-adjust'); 62 71 63 72 $applied = $api->getAppliedPatches(); 64 73 if ($applied === null) { ··· 187 196 } 188 197 } 189 198 190 - return 0; 199 + $console = PhutilConsole::getConsole(); 200 + if ($no_adjust || $init_only || $apply_only) { 201 + $console->writeOut( 202 + "%s\n", 203 + pht('Declining to apply storage adjustments.')); 204 + return 0; 205 + } else { 206 + return $this->adjustSchemata($is_force, $unsafe = false, $is_dry); 207 + } 191 208 } 192 209 193 210 }

+607

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php

··· 25 25 return $this->api; 26 26 } 27 27 28 + private function loadSchemata() { 29 + $query = id(new PhabricatorConfigSchemaQuery()) 30 + ->setAPI($this->getAPI()); 31 + 32 + $actual = $query->loadActualSchema(); 33 + $expect = $query->loadExpectedSchema(); 34 + $comp = $query->buildComparisonSchema($expect, $actual); 35 + 36 + return array($comp, $expect, $actual); 37 + } 38 + 39 + protected function adjustSchemata($force, $unsafe, $dry_run) { 40 + $console = PhutilConsole::getConsole(); 41 + 42 + $console->writeOut( 43 + "%s\n", 44 + pht('Verifying database schemata...')); 45 + 46 + list($adjustments, $errors) = $this->findAdjustments(); 47 + $api = $this->getAPI(); 48 + 49 + if (!$adjustments) { 50 + $console->writeOut( 51 + "%s\n", 52 + pht('Found no adjustments for schemata.')); 53 + 54 + return $this->printErrors($errors, 0); 55 + } 56 + 57 + if (!$force && !$api->isCharacterSetAvailable('utf8mb4')) { 58 + $message = pht( 59 + "You have an old version of MySQL (older than 5.5) which does not ". 60 + "support the utf8mb4 character set. If you apply adjustments now ". 61 + "and later update MySQL to 5.5 or newer, you'll need to apply ". 62 + "adjustments again (and they will take a long time).\n\n". 63 + "You can exit this workflow, update MySQL now, and then run this ". 64 + "workflow again. This is recommended, but may cause a lot of downtime ". 65 + "right now.\n\n". 66 + "You can exit this workflow, continue using Phabricator without ". 67 + "applying adjustments, update MySQL at a later date, and then run ". 68 + "this workflow again. This is also a good approach, and will let you ". 69 + "delay downtime until later.\n\n". 70 + "You can proceed with this workflow, and then optionally update ". 71 + "MySQL at a later date. After you do, you'll need to apply ". 72 + "adjustments again.\n\n". 73 + "For more information, see \"Managing Storage Adjustments\" in ". 74 + "the documentation."); 75 + 76 + $console->writeOut( 77 + "\n**<bg:yellow> %s </bg>**\n\n%s\n", 78 + pht('OLD MySQL VERSION'), 79 + phutil_console_wrap($message)); 80 + 81 + $prompt = pht('Continue with old MySQL version?'); 82 + if (!phutil_console_confirm($prompt, $default_no = true)) { 83 + return; 84 + } 85 + } 86 + 87 + $table = id(new PhutilConsoleTable()) 88 + ->addColumn('database', array('title' => pht('Database'))) 89 + ->addColumn('table', array('title' => pht('Table'))) 90 + ->addColumn('name', array('title' => pht('Name'))) 91 + ->addColumn('info', array('title' => pht('Issues'))); 92 + 93 + foreach ($adjustments as $adjust) { 94 + $info = array(); 95 + foreach ($adjust['issues'] as $issue) { 96 + $info[] = PhabricatorConfigStorageSchema::getIssueName($issue); 97 + } 98 + 99 + $table->addRow(array( 100 + 'database' => $adjust['database'], 101 + 'table' => idx($adjust, 'table'), 102 + 'name' => idx($adjust, 'name'), 103 + 'info' => implode(', ', $info), 104 + )); 105 + } 106 + 107 + $console->writeOut("\n\n"); 108 + 109 + $table->draw(); 110 + 111 + if ($dry_run) { 112 + $console->writeOut( 113 + "%s\n", 114 + pht('DRYRUN: Would apply adjustments.')); 115 + return 0; 116 + } else if (!$force) { 117 + $console->writeOut( 118 + "\n%s\n", 119 + pht( 120 + "Found %s issues(s) with schemata, detailed above.\n\n". 121 + "You can review issues in more detail from the web interface, ". 122 + "in Config > Database Status. To better understand the adjustment ". 123 + "workflow, see \"Managing Storage Adjustments\" in the ". 124 + "documentation.\n\n". 125 + "MySQL needs to copy table data to make some adjustments, so these ". 126 + "migrations may take some time.", 127 + new PhutilNumber(count($adjustments)))); 128 + 129 + $prompt = pht('Fix these schema issues?'); 130 + if (!phutil_console_confirm($prompt, $default_no = true)) { 131 + return 1; 132 + } 133 + } 134 + 135 + $console->writeOut( 136 + "%s\n", 137 + pht('Dropping caches, for faster migrations...')); 138 + 139 + $root = dirname(phutil_get_library_root('phabricator')); 140 + $bin = $root.'/bin/cache'; 141 + phutil_passthru('%s purge --purge-all', $bin); 142 + 143 + $console->writeOut( 144 + "%s\n", 145 + pht('Fixing schema issues...')); 146 + 147 + $conn = $api->getConn(null); 148 + 149 + if ($unsafe) { 150 + queryfx($conn, 'SET SESSION sql_mode = %s', ''); 151 + } else { 152 + queryfx($conn, 'SET SESSION sql_mode = %s', 'STRICT_ALL_TABLES'); 153 + } 154 + 155 + $failed = array(); 156 + 157 + // We make changes in several phases. 158 + $phases = array( 159 + // Drop surplus autoincrements. This allows us to drop primary keys on 160 + // autoincrement columns. 161 + 'drop_auto', 162 + 163 + // Drop all keys we're going to adjust. This prevents them from 164 + // interfering with column changes. 165 + 'drop_keys', 166 + 167 + // Apply all database, table, and column changes. 168 + 'main', 169 + 170 + // Restore adjusted keys. 171 + 'add_keys', 172 + 173 + // Add missing autoincrements. 174 + 'add_auto', 175 + ); 176 + 177 + $bar = id(new PhutilConsoleProgressBar()) 178 + ->setTotal(count($adjustments) * count($phases)); 179 + 180 + foreach ($phases as $phase) { 181 + foreach ($adjustments as $adjust) { 182 + try { 183 + switch ($adjust['kind']) { 184 + case 'database': 185 + if ($phase == 'main') { 186 + queryfx( 187 + $conn, 188 + 'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s', 189 + $adjust['database'], 190 + $adjust['charset'], 191 + $adjust['collation']); 192 + } 193 + break; 194 + case 'table': 195 + if ($phase == 'main') { 196 + queryfx( 197 + $conn, 198 + 'ALTER TABLE %T.%T COLLATE = %s', 199 + $adjust['database'], 200 + $adjust['table'], 201 + $adjust['collation']); 202 + } 203 + break; 204 + case 'column': 205 + $apply = false; 206 + $auto = false; 207 + $new_auto = idx($adjust, 'auto'); 208 + if ($phase == 'drop_auto') { 209 + if ($new_auto === false) { 210 + $apply = true; 211 + $auto = false; 212 + } 213 + } else if ($phase == 'main') { 214 + $apply = true; 215 + if ($new_auto === false) { 216 + $auto = false; 217 + } else { 218 + $auto = $adjust['is_auto']; 219 + } 220 + } else if ($phase == 'add_auto') { 221 + if ($new_auto === true) { 222 + $apply = true; 223 + $auto = true; 224 + } 225 + } 226 + 227 + if ($apply) { 228 + $parts = array(); 229 + 230 + if ($auto) { 231 + $parts[] = qsprintf( 232 + $conn, 233 + 'AUTO_INCREMENT'); 234 + } 235 + 236 + if ($adjust['charset']) { 237 + $parts[] = qsprintf( 238 + $conn, 239 + 'CHARACTER SET %Q COLLATE %Q', 240 + $adjust['charset'], 241 + $adjust['collation']); 242 + } 243 + 244 + queryfx( 245 + $conn, 246 + 'ALTER TABLE %T.%T MODIFY %T %Q %Q %Q', 247 + $adjust['database'], 248 + $adjust['table'], 249 + $adjust['name'], 250 + $adjust['type'], 251 + implode(' ', $parts), 252 + $adjust['nullable'] ? 'NULL' : 'NOT NULL'); 253 + } 254 + break; 255 + case 'key': 256 + if (($phase == 'drop_keys') && $adjust['exists']) { 257 + if ($adjust['name'] == 'PRIMARY') { 258 + $key_name = 'PRIMARY KEY'; 259 + } else { 260 + $key_name = qsprintf($conn, 'KEY %T', $adjust['name']); 261 + } 262 + 263 + queryfx( 264 + $conn, 265 + 'ALTER TABLE %T.%T DROP %Q', 266 + $adjust['database'], 267 + $adjust['table'], 268 + $key_name); 269 + } 270 + 271 + if (($phase == 'add_keys') && $adjust['keep']) { 272 + // Different keys need different creation syntax. Notable 273 + // special cases are primary keys and fulltext keys. 274 + if ($adjust['name'] == 'PRIMARY') { 275 + $key_name = 'PRIMARY KEY'; 276 + } else if ($adjust['indexType'] == 'FULLTEXT') { 277 + $key_name = qsprintf($conn, 'FULLTEXT %T', $adjust['name']); 278 + } else { 279 + if ($adjust['unique']) { 280 + $key_name = qsprintf( 281 + $conn, 282 + 'UNIQUE KEY %T', 283 + $adjust['name']); 284 + } else { 285 + $key_name = qsprintf( 286 + $conn, 287 + '/* NONUNIQUE */ KEY %T', 288 + $adjust['name']); 289 + } 290 + } 291 + 292 + queryfx( 293 + $conn, 294 + 'ALTER TABLE %T.%T ADD %Q (%Q)', 295 + $adjust['database'], 296 + $adjust['table'], 297 + $key_name, 298 + implode(', ', $adjust['columns'])); 299 + } 300 + break; 301 + default: 302 + throw new Exception( 303 + pht('Unknown schema adjustment kind "%s"!', $adjust['kind'])); 304 + } 305 + } catch (AphrontQueryException $ex) { 306 + $failed[] = array($adjust, $ex); 307 + } 308 + $bar->update(1); 309 + } 310 + } 311 + $bar->done(); 312 + 313 + if (!$failed) { 314 + $console->writeOut( 315 + "%s\n", 316 + pht('Completed fixing all schema issues.')); 317 + 318 + $err = 0; 319 + } else { 320 + $table = id(new PhutilConsoleTable()) 321 + ->addColumn('target', array('title' => pht('Target'))) 322 + ->addColumn('error', array('title' => pht('Error'))); 323 + 324 + foreach ($failed as $failure) { 325 + list($adjust, $ex) = $failure; 326 + 327 + $pieces = array_select_keys( 328 + $adjust, 329 + array('database', 'table', 'name')); 330 + $pieces = array_filter($pieces); 331 + $target = implode('.', $pieces); 332 + 333 + $table->addRow( 334 + array( 335 + 'target' => $target, 336 + 'error' => $ex->getMessage(), 337 + )); 338 + } 339 + 340 + $console->writeOut("\n"); 341 + $table->draw(); 342 + $console->writeOut( 343 + "\n%s\n", 344 + pht('Failed to make some schema adjustments, detailed above.')); 345 + $console->writeOut( 346 + "%s\n", 347 + pht( 348 + 'For help troubleshooting adjustments, see "Managing Storage '. 349 + 'Adjustments" in the documentation.')); 350 + 351 + $err = 1; 352 + } 353 + 354 + return $this->printErrors($errors, $err); 355 + } 356 + 357 + private function findAdjustments() { 358 + list($comp, $expect, $actual) = $this->loadSchemata(); 359 + 360 + $issue_charset = PhabricatorConfigStorageSchema::ISSUE_CHARSET; 361 + $issue_collation = PhabricatorConfigStorageSchema::ISSUE_COLLATION; 362 + $issue_columntype = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE; 363 + $issue_surpluskey = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY; 364 + $issue_missingkey = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY; 365 + $issue_columns = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS; 366 + $issue_unique = PhabricatorConfigStorageSchema::ISSUE_UNIQUE; 367 + $issue_longkey = PhabricatorConfigStorageSchema::ISSUE_LONGKEY; 368 + $issue_auto = PhabricatorConfigStorageSchema::ISSUE_AUTOINCREMENT; 369 + 370 + $adjustments = array(); 371 + $errors = array(); 372 + foreach ($comp->getDatabases() as $database_name => $database) { 373 + foreach ($this->findErrors($database) as $issue) { 374 + $errors[] = array( 375 + 'database' => $database_name, 376 + 'issue' => $issue, 377 + ); 378 + } 379 + 380 + $expect_database = $expect->getDatabase($database_name); 381 + $actual_database = $actual->getDatabase($database_name); 382 + 383 + if (!$expect_database || !$actual_database) { 384 + // If there's a real issue here, skip this stuff. 385 + continue; 386 + } 387 + 388 + $issues = array(); 389 + if ($database->hasIssue($issue_charset)) { 390 + $issues[] = $issue_charset; 391 + } 392 + if ($database->hasIssue($issue_collation)) { 393 + $issues[] = $issue_collation; 394 + } 395 + 396 + if ($issues) { 397 + $adjustments[] = array( 398 + 'kind' => 'database', 399 + 'database' => $database_name, 400 + 'issues' => $issues, 401 + 'charset' => $expect_database->getCharacterSet(), 402 + 'collation' => $expect_database->getCollation(), 403 + ); 404 + } 405 + 406 + foreach ($database->getTables() as $table_name => $table) { 407 + foreach ($this->findErrors($table) as $issue) { 408 + $errors[] = array( 409 + 'database' => $database_name, 410 + 'table' => $table_name, 411 + 'issue' => $issue, 412 + ); 413 + } 414 + 415 + $expect_table = $expect_database->getTable($table_name); 416 + $actual_table = $actual_database->getTable($table_name); 417 + 418 + if (!$expect_table || !$actual_table) { 419 + continue; 420 + } 421 + 422 + $issues = array(); 423 + if ($table->hasIssue($issue_collation)) { 424 + $issues[] = $issue_collation; 425 + } 426 + 427 + if ($issues) { 428 + $adjustments[] = array( 429 + 'kind' => 'table', 430 + 'database' => $database_name, 431 + 'table' => $table_name, 432 + 'issues' => $issues, 433 + 'collation' => $expect_table->getCollation(), 434 + ); 435 + } 436 + 437 + foreach ($table->getColumns() as $column_name => $column) { 438 + foreach ($this->findErrors($column) as $issue) { 439 + $errors[] = array( 440 + 'database' => $database_name, 441 + 'table' => $table_name, 442 + 'name' => $column_name, 443 + 'issue' => $issue, 444 + ); 445 + } 446 + 447 + $expect_column = $expect_table->getColumn($column_name); 448 + $actual_column = $actual_table->getColumn($column_name); 449 + 450 + if (!$expect_column || !$actual_column) { 451 + continue; 452 + } 453 + 454 + $issues = array(); 455 + if ($column->hasIssue($issue_collation)) { 456 + $issues[] = $issue_collation; 457 + } 458 + if ($column->hasIssue($issue_charset)) { 459 + $issues[] = $issue_charset; 460 + } 461 + if ($column->hasIssue($issue_columntype)) { 462 + $issues[] = $issue_columntype; 463 + } 464 + if ($column->hasIssue($issue_auto)) { 465 + $issues[] = $issue_auto; 466 + } 467 + 468 + if ($issues) { 469 + if ($expect_column->getCharacterSet() === null) { 470 + // For non-text columns, we won't be specifying a collation or 471 + // character set. 472 + $charset = null; 473 + $collation = null; 474 + } else { 475 + $charset = $expect_column->getCharacterSet(); 476 + $collation = $expect_column->getCollation(); 477 + } 478 + 479 + $adjustment = array( 480 + 'kind' => 'column', 481 + 'database' => $database_name, 482 + 'table' => $table_name, 483 + 'name' => $column_name, 484 + 'issues' => $issues, 485 + 'collation' => $collation, 486 + 'charset' => $charset, 487 + 'type' => $expect_column->getColumnType(), 488 + 489 + // NOTE: We don't adjust column nullability because it is 490 + // dangerous, so always use the current nullability. 491 + 'nullable' => $actual_column->getNullable(), 492 + 493 + // NOTE: This always stores the current value, because we have 494 + // to make these updates separately. 495 + 'is_auto' => $actual_column->getAutoIncrement(), 496 + ); 497 + 498 + if ($column->hasIssue($issue_auto)) { 499 + $adjustment['auto'] = $expect_column->getAutoIncrement(); 500 + } 501 + 502 + $adjustments[] = $adjustment; 503 + } 504 + } 505 + 506 + foreach ($table->getKeys() as $key_name => $key) { 507 + foreach ($this->findErrors($key) as $issue) { 508 + $errors[] = array( 509 + 'database' => $database_name, 510 + 'table' => $table_name, 511 + 'name' => $key_name, 512 + 'issue' => $issue, 513 + ); 514 + } 515 + 516 + $expect_key = $expect_table->getKey($key_name); 517 + $actual_key = $actual_table->getKey($key_name); 518 + 519 + $issues = array(); 520 + $keep_key = true; 521 + if ($key->hasIssue($issue_surpluskey)) { 522 + $issues[] = $issue_surpluskey; 523 + $keep_key = false; 524 + } 525 + 526 + if ($key->hasIssue($issue_missingkey)) { 527 + $issues[] = $issue_missingkey; 528 + } 529 + 530 + if ($key->hasIssue($issue_columns)) { 531 + $issues[] = $issue_columns; 532 + } 533 + 534 + if ($key->hasIssue($issue_unique)) { 535 + $issues[] = $issue_unique; 536 + } 537 + 538 + // NOTE: We can't really fix this, per se, but we may need to remove 539 + // the key to change the column type. In the best case, the new 540 + // column type won't be overlong and recreating the key really will 541 + // fix the issue. In the worst case, we get the right column type and 542 + // lose the key, which is still better than retaining the key having 543 + // the wrong column type. 544 + if ($key->hasIssue($issue_longkey)) { 545 + $issues[] = $issue_longkey; 546 + } 547 + 548 + if ($issues) { 549 + $adjustment = array( 550 + 'kind' => 'key', 551 + 'database' => $database_name, 552 + 'table' => $table_name, 553 + 'name' => $key_name, 554 + 'issues' => $issues, 555 + 'exists' => (bool)$actual_key, 556 + 'keep' => $keep_key, 557 + ); 558 + 559 + if ($keep_key) { 560 + $adjustment += array( 561 + 'columns' => $expect_key->getColumnNames(), 562 + 'unique' => $expect_key->getUnique(), 563 + 'indexType' => $expect_key->getIndexType(), 564 + ); 565 + } 566 + 567 + $adjustments[] = $adjustment; 568 + } 569 + } 570 + } 571 + } 572 + 573 + return array($adjustments, $errors); 574 + } 575 + 576 + private function findErrors(PhabricatorConfigStorageSchema $schema) { 577 + $result = array(); 578 + foreach ($schema->getLocalIssues() as $issue) { 579 + $status = PhabricatorConfigStorageSchema::getIssueStatus($issue); 580 + if ($status == PhabricatorConfigStorageSchema::STATUS_FAIL) { 581 + $result[] = $issue; 582 + } 583 + } 584 + return $result; 585 + } 586 + 587 + private function printErrors(array $errors, $default_return) { 588 + if (!$errors) { 589 + return $default_return; 590 + } 591 + 592 + $console = PhutilConsole::getConsole(); 593 + 594 + $table = id(new PhutilConsoleTable()) 595 + ->addColumn('target', array('title' => pht('Target'))) 596 + ->addColumn('error', array('title' => pht('Error'))); 597 + 598 + foreach ($errors as $error) { 599 + $pieces = array_select_keys( 600 + $error, 601 + array('database', 'table', 'name')); 602 + $pieces = array_filter($pieces); 603 + $target = implode('.', $pieces); 604 + 605 + $name = PhabricatorConfigStorageSchema::getIssueName($error['issue']); 606 + 607 + $table->addRow( 608 + array( 609 + 'target' => $target, 610 + 'error' => $name, 611 + )); 612 + } 613 + 614 + $console->writeOut("\n"); 615 + $table->draw(); 616 + $console->writeOut("\n"); 617 + 618 + $message = pht( 619 + "The schemata have serious errors (detailed above) which the adjustment ". 620 + "workflow can not fix.\n\n". 621 + "If you are not developing Phabricator itself, report this issue to ". 622 + "the upstream.\n\n". 623 + "If you are developing Phabricator, these errors usually indicate that ". 624 + "your schema specifications do not agree with the schemata your code ". 625 + "actually builds."); 626 + 627 + $console->writeOut( 628 + "**<bg:red> %s </bg>**\n\n%s\n", 629 + pht('SCHEMATA ERRORS'), 630 + phutil_console_wrap($message)); 631 + 632 + return 2; 633 + } 634 + 28 635 }

Configure Feed

Configure Feed