Add "bin/storage optimize" to run OPTIMIZE TABLE on everything

Summary:
Even with `innodb_file_per_table` enabled, individual table files on disk don't normally shrink.

For most tables, like `maniphest_task`, this is fine, since the data in the table normally never shrinks, or only shinks a tiny amount.

However, some tables (like the "worker" and "daemon" tables) grow very large during a huge import but most of the data is later deleted by garbage collection. In these cases, this lost space can be reclaimed by running `OPTIMIZE TABLE` on the tables.

Add a script to `OPTIMIZE TABLE` every table.

My primary goal here is just to reduce storage pressure on `db001` since there are a couple of "import the linux kernel" installs on that host wasting a bunch of space. We're not in any trouble, but this should buy us a good chunk of headroom.

Test Plan: Ran `bin/storage optimize` locally and manually ran `OPTIMIZE TABLE` in production, saw tables get optimized.

Reviewers: chad

Reviewed By: chad

Subscribers: cspeckmim

Differential Revision: https://secure.phabricator.com/D17640

epriestley 9 years ago d1421bc3 77076857

+88

2 changed files

expand all

src

__phutil_library_map__.php

infrastructure

storage

management

workflow

PhabricatorStorageManagementOptimizeWorkflow.php

src/__phutil_library_map__.php

··· 3940 3940 'PhabricatorStorageManagementDatabasesWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php', 3941 3941 'PhabricatorStorageManagementDestroyWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php', 3942 3942 'PhabricatorStorageManagementDumpWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php', 3943 + 'PhabricatorStorageManagementOptimizeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementOptimizeWorkflow.php', 3943 3944 'PhabricatorStorageManagementPartitionWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementPartitionWorkflow.php', 3944 3945 'PhabricatorStorageManagementProbeWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php', 3945 3946 'PhabricatorStorageManagementQuickstartWorkflow' => 'infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php', ··· 9286 9287 'PhabricatorStorageManagementDatabasesWorkflow' => 'PhabricatorStorageManagementWorkflow', 9287 9288 'PhabricatorStorageManagementDestroyWorkflow' => 'PhabricatorStorageManagementWorkflow', 9288 9289 'PhabricatorStorageManagementDumpWorkflow' => 'PhabricatorStorageManagementWorkflow', 9290 + 'PhabricatorStorageManagementOptimizeWorkflow' => 'PhabricatorStorageManagementWorkflow', 9289 9291 'PhabricatorStorageManagementPartitionWorkflow' => 'PhabricatorStorageManagementWorkflow', 9290 9292 'PhabricatorStorageManagementProbeWorkflow' => 'PhabricatorStorageManagementWorkflow', 9291 9293 'PhabricatorStorageManagementQuickstartWorkflow' => 'PhabricatorStorageManagementWorkflow',

+86

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementOptimizeWorkflow.php

··· 1 + <?php 2 + 3 + final class PhabricatorStorageManagementOptimizeWorkflow 4 + extends PhabricatorStorageManagementWorkflow { 5 + 6 + protected function didConstruct() { 7 + $this 8 + ->setName('optimize') 9 + ->setExamples('**optimize**') 10 + ->setSynopsis(pht('Run "OPTIMIZE TABLE" on tables to reclaim space.')); 11 + } 12 + 13 + public function didExecute(PhutilArgumentParser $args) { 14 + $api = $this->getSingleAPI(); 15 + $conn = $api->getConn(null); 16 + 17 + $patches = $this->getPatches(); 18 + $databases = $api->getDatabaseList($patches, true); 19 + 20 + $total_bytes = 0; 21 + foreach ($databases as $database) { 22 + queryfx($conn, 'USE %C', $database); 23 + 24 + $tables = queryfx_all($conn, 'SHOW TABLE STATUS'); 25 + foreach ($tables as $table) { 26 + $table_name = $table['Name']; 27 + $old_bytes = 28 + $table['Data_length'] + 29 + $table['Index_length'] + 30 + $table['Data_free']; 31 + 32 + $this->logInfo( 33 + pht('OPTIMIZE'), 34 + pht( 35 + 'Optimizing table "%s"."%s"...', 36 + $database, 37 + $table_name)); 38 + 39 + $t_start = microtime(true); 40 + queryfx( 41 + $conn, 42 + 'OPTIMIZE TABLE %T', 43 + $table_name); 44 + $t_end = microtime(true); 45 + 46 + $status = queryfx_one( 47 + $conn, 48 + 'SHOW TABLE STATUS LIKE %s', 49 + $table_name); 50 + 51 + $new_bytes = 52 + $status['Data_length'] + 53 + $status['Index_length'] + 54 + $status['Data_free']; 55 + 56 + $duration_ms = (int)(1000 * ($t_end - $t_start)); 57 + 58 + if ($old_bytes > $new_bytes) { 59 + $this->logOkay( 60 + pht('DONE'), 61 + pht( 62 + 'Compacted table by %s in %sms.', 63 + phutil_format_bytes($old_bytes - $new_bytes), 64 + new PhutilNumber($duration_ms))); 65 + } else { 66 + $this->logInfo( 67 + pht('DONE'), 68 + pht( 69 + 'Optimized table (in %sms) but it had little effect.', 70 + new PhutilNumber($duration_ms))); 71 + } 72 + 73 + $total_bytes += ($old_bytes - $new_bytes); 74 + } 75 + } 76 + 77 + $this->logOkay( 78 + pht('OPTIMIZED'), 79 + pht( 80 + 'Completed optimizations, reclaimed %s of disk space.', 81 + phutil_format_bytes($total_bytes))); 82 + 83 + return 0; 84 + } 85 + 86 + }

Configure Feed

Configure Feed