@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Add a publish cache for the Diviner static publisher
Summary: Keep track of what we've written to disk, and regenerate only new documents.
Test Plan: Changed a small number of files, saw that number of files get regenerated. Ran with "--clean" and saw everything regenerate.
Reviewers: btrahan, vrana, chad
Reviewed By: chad
CC: aran
Maniphest Tasks: T988
Differential Revision: https://secure.phabricator.com/D4897
+167
-39
+4
src/__phutil_library_map__.php
+4
src/__phutil_library_map__.php
···
468
468
'DivinerAtomizeWorkflow' => 'applications/diviner/workflow/DivinerAtomizeWorkflow.php',
469
469
'DivinerAtomizer' => 'applications/diviner/atomizer/DivinerAtomizer.php',
470
470
'DivinerDefaultRenderer' => 'applications/diviner/renderer/DivinerDefaultRenderer.php',
471
+
'DivinerDiskCache' => 'applications/diviner/cache/DivinerDiskCache.php',
471
472
'DivinerFileAtomizer' => 'applications/diviner/atomizer/DivinerFileAtomizer.php',
472
473
'DivinerGenerateWorkflow' => 'applications/diviner/workflow/DivinerGenerateWorkflow.php',
473
474
'DivinerListController' => 'applications/diviner/controller/DivinerListController.php',
475
+
'DivinerPublishCache' => 'applications/diviner/cache/DivinerPublishCache.php',
474
476
'DivinerPublisher' => 'applications/diviner/publisher/DivinerPublisher.php',
475
477
'DivinerRenderer' => 'applications/diviner/renderer/DivinerRenderer.php',
476
478
'DivinerStaticPublisher' => 'applications/diviner/publisher/DivinerStaticPublisher.php',
···
1965
1967
'DiffusionURITestCase' => 'ArcanistPhutilTestCase',
1966
1968
'DiffusionView' => 'AphrontView',
1967
1969
'DivinerArticleAtomizer' => 'DivinerAtomizer',
1970
+
'DivinerAtomCache' => 'DivinerDiskCache',
1968
1971
'DivinerAtomizeWorkflow' => 'DivinerWorkflow',
1969
1972
'DivinerDefaultRenderer' => 'DivinerRenderer',
1970
1973
'DivinerFileAtomizer' => 'DivinerAtomizer',
1971
1974
'DivinerGenerateWorkflow' => 'DivinerWorkflow',
1972
1975
'DivinerListController' => 'PhabricatorController',
1976
+
'DivinerPublishCache' => 'DivinerDiskCache',
1973
1977
'DivinerStaticPublisher' => 'DivinerPublisher',
1974
1978
'DivinerWorkflow' => 'PhutilArgumentWorkflow',
1975
1979
'DrydockAllocatorWorker' => 'PhabricatorWorker',
+3
-32
src/applications/diviner/cache/DivinerAtomCache.php
+3
-32
src/applications/diviner/cache/DivinerAtomCache.php
···
1
1
<?php
2
2
3
-
final class DivinerAtomCache {
4
-
5
-
private $cache;
3
+
final class DivinerAtomCache extends DivinerDiskCache {
6
4
7
5
private $fileHashMap;
8
6
private $atomMap;
···
15
13
private $writeAtoms = array();
16
14
17
15
public function __construct($cache_directory) {
18
-
$dir_cache = id(new PhutilKeyValueCacheDirectory())
19
-
->setCacheDirectory($cache_directory);
20
-
$profiled_cache = id(new PhutilKeyValueCacheProfiler($dir_cache))
21
-
->setProfiler(PhutilServiceProfiler::getInstance())
22
-
->setName('diviner-atom-cache');
23
-
$this->cache = $profiled_cache;
24
-
}
25
-
26
-
private function getCache() {
27
-
return $this->cache;
16
+
return parent::__construct($cache_directory, 'diviner-atom-cache');
28
17
}
29
18
30
19
public function delete() {
31
-
$this->getCache()->destroyCache();
20
+
parent::delete();
32
21
$this->fileHashMap = null;
33
22
$this->atomMap = null;
34
23
$this->atoms = array();
35
24
36
25
return $this;
37
26
}
38
-
39
-
/**
40
-
* Convert a long-form hash key like `ccbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaN` into
41
-
* a shortened directory form, like `cc/bb/aaaaaaaaN`. In conjunction with
42
-
* @{class:PhutilKeyValueCacheDirectory}, this gives us nice directories
43
-
* inside .divinercache instead of a million hash files with huge names at
44
-
* top level.
45
-
*/
46
-
private function getHashKey($hash) {
47
-
return implode(
48
-
'/',
49
-
array(
50
-
substr($hash, 0, 2),
51
-
substr($hash, 2, 2),
52
-
substr($hash, 4, 8),
53
-
));
54
-
}
55
-
56
27
57
28
/* -( File Hash Map )------------------------------------------------------ */
58
29
+42
src/applications/diviner/cache/DivinerDiskCache.php
+42
src/applications/diviner/cache/DivinerDiskCache.php
···
1
+
<?php
2
+
3
+
abstract class DivinerDiskCache {
4
+
5
+
private $cache;
6
+
7
+
public function __construct($cache_directory, $name) {
8
+
$dir_cache = id(new PhutilKeyValueCacheDirectory())
9
+
->setCacheDirectory($cache_directory);
10
+
$profiled_cache = id(new PhutilKeyValueCacheProfiler($dir_cache))
11
+
->setProfiler(PhutilServiceProfiler::getInstance())
12
+
->setName($name);
13
+
$this->cache = $profiled_cache;
14
+
}
15
+
16
+
protected function getCache() {
17
+
return $this->cache;
18
+
}
19
+
20
+
public function delete() {
21
+
$this->getCache()->destroyCache();
22
+
return $this;
23
+
}
24
+
25
+
/**
26
+
* Convert a long-form hash key like `ccbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaN` into
27
+
* a shortened directory form, like `cc/bb/aaaaaaaaN`. In conjunction with
28
+
* @{class:PhutilKeyValueCacheDirectory}, this gives us nice directories
29
+
* inside .divinercache instead of a million hash files with huge names at
30
+
* top level.
31
+
*/
32
+
protected function getHashKey($hash) {
33
+
return implode(
34
+
'/',
35
+
array(
36
+
substr($hash, 0, 2),
37
+
substr($hash, 2, 2),
38
+
substr($hash, 4, 8),
39
+
));
40
+
}
41
+
42
+
}
+45
src/applications/diviner/cache/DivinerPublishCache.php
+45
src/applications/diviner/cache/DivinerPublishCache.php
···
1
+
<?php
2
+
3
+
final class DivinerPublishCache extends DivinerDiskCache {
4
+
5
+
private $pathMap;
6
+
7
+
public function __construct($cache_directory) {
8
+
return parent::__construct($cache_directory, 'diviner-publish-cache');
9
+
}
10
+
11
+
12
+
/* -( Path Map )----------------------------------------------------------- */
13
+
14
+
15
+
public function getPathMap() {
16
+
if ($this->pathMap === null) {
17
+
$this->pathMap = $this->getCache()->getKey('path', array());
18
+
}
19
+
return $this->pathMap;
20
+
}
21
+
22
+
public function writePathMap() {
23
+
$this->getCache()->setKey('path', $this->getPathMap());
24
+
}
25
+
26
+
public function getAtomPathsFromCache($hash) {
27
+
return idx($this->getPathMap(), $hash, array());
28
+
}
29
+
30
+
public function removeAtomPathsFromCache($hash) {
31
+
$map = $this->getPathMap();
32
+
unset($map[$hash]);
33
+
$this->pathMap = $map;
34
+
return $this;
35
+
}
36
+
37
+
public function addAtomPathsToCache($hash, array $paths) {
38
+
$map = $this->getPathMap();
39
+
$map[$hash] = $paths;
40
+
$this->pathMap = $map;
41
+
return $this;
42
+
}
43
+
44
+
45
+
}
+4
-1
src/applications/diviner/publisher/DivinerPublisher.php
+4
-1
src/applications/diviner/publisher/DivinerPublisher.php
···
113
113
$deleted = array_diff_key($existing_map, $hashes_map);
114
114
$created = array_diff_key($hashes_map, $existing_map);
115
115
116
+
echo pht('Deleting %d documents.', count($deleted))."\n";
117
+
$this->deleteDocumentsByHash(array_keys($deleted));
118
+
119
+
echo pht('Creating %d documents.', count($created))."\n";
116
120
$this->createDocumentsByHash(array_keys($created));
117
-
$this->deleteDocumentsByHash(array_keys($deleted));
118
121
}
119
122
120
123
protected function shouldGenerateDocumentForAtom(DivinerAtom $atom) {
+69
-6
src/applications/diviner/publisher/DivinerStaticPublisher.php
+69
-6
src/applications/diviner/publisher/DivinerStaticPublisher.php
···
2
2
3
3
final class DivinerStaticPublisher extends DivinerPublisher {
4
4
5
+
private $publishCache;
6
+
7
+
private function getPublishCache() {
8
+
if (!$this->publishCache) {
9
+
$dir = implode(
10
+
DIRECTORY_SEPARATOR,
11
+
array(
12
+
$this->getConfig('root'),
13
+
'.divinercache',
14
+
$this->getConfig('name'),
15
+
'static',
16
+
));
17
+
$this->publishCache = new DivinerPublishCache($dir);
18
+
}
19
+
20
+
return $this->publishCache;
21
+
}
22
+
5
23
protected function loadAllPublishedHashes() {
6
-
return array();
24
+
return array_keys($this->getPublishCache()->getPathMap());
7
25
}
8
26
9
27
protected function deleteDocumentsByHash(array $hashes) {
10
-
return;
28
+
$root = $this->getConfig('root');
29
+
$cache = $this->getPublishCache();
30
+
31
+
foreach ($hashes as $hash) {
32
+
$paths = $cache->getAtomPathsFromCache($hash);
33
+
foreach ($paths as $path) {
34
+
$abs = $root.DIRECTORY_SEPARATOR.$path;
35
+
Filesystem::remove($abs);
36
+
37
+
// If the parent directory is now empty, clean it up.
38
+
$dir = dirname($abs);
39
+
while (true) {
40
+
if (!Filesystem::isDescendant($dir, $root)) {
41
+
// Directory is outside of the root.
42
+
break;
43
+
}
44
+
if (Filesystem::listDirectory($dir)) {
45
+
// Directory is not empty.
46
+
break;
47
+
}
48
+
49
+
Filesystem::remove($dir);
50
+
$dir = dirname($dir);
51
+
}
52
+
}
53
+
54
+
$cache->removeAtomPathsFromCache($hash);
55
+
$cache->deleteRenderCache($hash);
56
+
}
57
+
58
+
$cache->writePathMap();
11
59
}
12
60
13
61
protected function createDocumentsByHash(array $hashes) {
62
+
$indexes = array();
63
+
14
64
foreach ($hashes as $hash) {
15
65
$atom = $this->getAtomFromGraphHash($hash);
16
66
17
-
if (!$this->shouldGenerateDocumentForAtom($atom)) {
18
-
continue;
67
+
$paths = array();
68
+
if ($this->shouldGenerateDocumentForAtom($atom)) {
69
+
$content = $this->getRenderer()->renderAtom($atom);
70
+
71
+
$this->writeDocument($atom, $content);
72
+
73
+
$paths[] = $this->getAtomRelativePath($atom);
74
+
if ($this->getAtomSimilarIndex($atom) !== null) {
75
+
$index = dirname($this->getAtomRelativePath($atom)).'index.html';
76
+
$indexes[$index] = $atom;
77
+
$paths[] = $index;
78
+
}
19
79
}
20
80
21
-
$content = $this->getRenderer()->renderAtom($atom);
22
-
$this->writeDocument($atom, $content);
81
+
$this->getPublishCache()->addAtomPathsToCache($hash, $paths);
23
82
}
83
+
84
+
$this->getPublishCache()->writePathMap();
24
85
}
25
86
26
87
private function writeDocument(DivinerAtom $atom, $content) {
···
32
93
}
33
94
34
95
Filesystem::writeFile($path.'index.html', $content);
96
+
97
+
return $this;
35
98
}
36
99
37
100
private function getAtomRelativePath(DivinerAtom $atom) {