@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Allow Fact analysis of commits
Summary: For immutable objects, just use the ID as a cursor.
Test Plan:
- Analyzed commits from an empty cursor.
- Checked that cursor was good.
- Pulled some more commits.
- Analyzed commits again, verified it only hit the new ones.
- Verified the graph of "Count of CMIT" looked reasonable.
Reviewers: vrana
Reviewed By: vrana
CC: aran
Maniphest Tasks: T1866
Differential Revision: https://secure.phabricator.com/D3656
+64
-28
+46
-25
src/applications/fact/extract/PhabricatorFactUpdateIterator.php
+46
-25
src/applications/fact/extract/PhabricatorFactUpdateIterator.php
···
33
33
public function __construct(LiskDAO $object) {
34
34
$this->set = new LiskDAOSet();
35
35
$this->object = $object->putInSet($this->set);
36
-
$this->position = '0:0';
37
36
}
38
37
39
38
public function setPosition($position) {
···
46
45
}
47
46
48
47
protected function getCursorFromObject($object) {
49
-
return $object->getDateModified().':'.$object->getID();
48
+
if ($object->hasProperty('dateModified')) {
49
+
return $object->getDateModified().':'.$object->getID();
50
+
} else {
51
+
return $object->getID();
52
+
}
50
53
}
51
54
52
55
public function key() {
···
54
57
}
55
58
56
59
protected function loadPage() {
57
-
list($after_epoch, $after_id) = explode(':', $this->cursor);
58
-
59
60
$this->set->clearSet();
60
61
61
-
// NOTE: We ignore recent updates because once we process an update we'll
62
-
// never process rows behind it again. We need to read only rows which
63
-
// we're sure no new rows will be inserted behind. If we read a row that
64
-
// was updated on the current second, another update later on in this second
65
-
// could affect an object with a lower ID, and we'd skip that update. To
66
-
// avoid this, just ignore any rows which have been updated in the last few
67
-
// seconds. This also reduces the amount of work we need to do if an object
68
-
// is repeatedly updated; we will just look at the end state without
69
-
// processing the intermediate states. Finally, this gives us reasonable
70
-
// protections against clock skew between the machine the daemon is running
71
-
// on and any machines performing writes.
62
+
if ($this->object->hasProperty('dateModified')) {
63
+
if ($this->cursor) {
64
+
list($after_epoch, $after_id) = explode(':', $this->cursor);
65
+
} else {
66
+
$after_epoch = 0;
67
+
$after_id = 0;
68
+
}
69
+
70
+
// NOTE: We ignore recent updates because once we process an update we'll
71
+
// never process rows behind it again. We need to read only rows which
72
+
// we're sure no new rows will be inserted behind. If we read a row that
73
+
// was updated on the current second, another update later on in this
74
+
// second could affect an object with a lower ID, and we'd skip that
75
+
// update. To avoid this, just ignore any rows which have been updated in
76
+
// the last few seconds. This also reduces the amount of work we need to
77
+
// do if an object is repeatedly updated; we will just look at the end
78
+
// state without processing the intermediate states. Finally, this gives
79
+
// us reasonable protections against clock skew between the machine the
80
+
// daemon is running on and any machines performing writes.
81
+
82
+
$page = $this->object->loadAllWhere(
83
+
'((dateModified > %d) OR (dateModified = %d AND id > %d))
84
+
AND (dateModified < %d - %d)
85
+
ORDER BY dateModified ASC, id ASC LIMIT %d',
86
+
$after_epoch,
87
+
$after_epoch,
88
+
$after_id,
89
+
time(),
90
+
$this->ignoreUpdatesDuration,
91
+
$this->getPageSize());
92
+
} else {
93
+
if ($this->cursor) {
94
+
$after_id = $this->cursor;
95
+
} else {
96
+
$after_id = 0;
97
+
}
72
98
73
-
$page = $this->object->loadAllWhere(
74
-
'((dateModified > %d) OR (dateModified = %d AND id > %d))
75
-
AND (dateModified < %d - %d)
76
-
ORDER BY dateModified ASC, id ASC LIMIT %d',
77
-
$after_epoch,
78
-
$after_epoch,
79
-
$after_id,
80
-
time(),
81
-
$this->ignoreUpdatesDuration,
82
-
$this->getPageSize());
99
+
$page = $this->object->loadAllWhere(
100
+
'id > %d ORDER BY id ASC LIMIT %d',
101
+
$after_id,
102
+
$this->getPageSize());
103
+
}
83
104
84
105
if ($page) {
85
106
$this->cursor = $this->getCursorFromObject(end($page));
+5
-2
src/applications/repository/storage/PhabricatorRepositoryCommit.php
+5
-2
src/applications/repository/storage/PhabricatorRepositoryCommit.php
···
102
102
return $result;
103
103
}
104
104
105
+
public function getDateCreated() {
106
+
// This is primarily to make analysis of commits with the Fact engine work.
107
+
return $this->getEpoch();
108
+
}
109
+
105
110
/**
106
111
* Synchronize a commit's overall audit status with the individual audit
107
112
* triggers.
···
143
148
144
149
return $this->setAuditStatus($status);
145
150
}
146
-
147
-
148
151
149
152
}
+13
-1
src/infrastructure/storage/lisk/LiskDAO.php
+13
-1
src/infrastructure/storage/lisk/LiskDAO.php
···
647
647
// already determined it's not valid. We don't need to check again.
648
648
continue;
649
649
}
650
-
$valid_properties[$k] = (bool)$this->checkProperty($k);
650
+
$valid_properties[$k] = $this->hasProperty($k);
651
651
if (!$valid_properties[$k]) {
652
652
continue;
653
653
}
···
893
893
$id_key = $this->getIDKeyForUse();
894
894
}
895
895
return $this->$id_key;
896
+
}
897
+
898
+
899
+
/**
900
+
* Test if a property exists.
901
+
*
902
+
* @param string Property name.
903
+
* @return bool True if the property exists.
904
+
* @task info
905
+
*/
906
+
public function hasProperty($property) {
907
+
return (bool)$this->checkProperty($property);
896
908
}
897
909
898
910