@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Implement a Git LFS link table and basic batch API
Summary:
Ref T7789. This implements:
- A new table to store the `<objectHash, filePHID>` relationship between Git LFS files and Phabricator file objects.
- A basic response to `batch` commands, which return actions for a list of files.
Test Plan:
Ran `git lfs push origin master`, got a little further than previously:
```
epriestley@orbital ~/dev/scratch/poemslocal $ git lfs push origin master
Git LFS: (2 of 1 files) 174.24 KB / 87.12 KB
Git LFS operation "upload/b7e0aeb82a03d627c6aa5fc1bbfd454b6789d9d9affc8607d40168fa18cf6c69" is not supported by this server.
Git LFS operation "upload/b7e0aeb82a03d627c6aa5fc1bbfd454b6789d9d9affc8607d40168fa18cf6c69" is not supported by this server.
```
With `GIT_TRACE=1`, this shows the batch part of the API going through.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T7789
Differential Revision: https://secure.phabricator.com/D15489
+304
-22
+11
resources/sql/autopatches/20160317.lfs.01.ref.sql
+11
resources/sql/autopatches/20160317.lfs.01.ref.sql
···
1
+
CREATE TABLE {$NAMESPACE}_repository.repository_gitlfsref (
2
+
id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
3
+
repositoryPHID VARBINARY(64) NOT NULL,
4
+
objectHash BINARY(64) NOT NULL,
5
+
byteSize BIGINT UNSIGNED NOT NULL,
6
+
authorPHID VARBINARY(64) NOT NULL,
7
+
filePHID VARBINARY(64) NOT NULL,
8
+
dateCreated INT UNSIGNED NOT NULL,
9
+
dateModified INT UNSIGNED NOT NULL,
10
+
UNIQUE KEY `key_hash` (repositoryPHID, objectHash)
11
+
) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};
+7
src/__phutil_library_map__.php
+7
src/__phutil_library_map__.php
···
3100
3100
'PhabricatorRepositoryEngine' => 'applications/repository/engine/PhabricatorRepositoryEngine.php',
3101
3101
'PhabricatorRepositoryGitCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositoryGitCommitChangeParserWorker.php',
3102
3102
'PhabricatorRepositoryGitCommitMessageParserWorker' => 'applications/repository/worker/commitmessageparser/PhabricatorRepositoryGitCommitMessageParserWorker.php',
3103
+
'PhabricatorRepositoryGitLFSRef' => 'applications/repository/storage/PhabricatorRepositoryGitLFSRef.php',
3104
+
'PhabricatorRepositoryGitLFSRefQuery' => 'applications/repository/query/PhabricatorRepositoryGitLFSRefQuery.php',
3103
3105
'PhabricatorRepositoryGraphCache' => 'applications/repository/graphcache/PhabricatorRepositoryGraphCache.php',
3104
3106
'PhabricatorRepositoryGraphStream' => 'applications/repository/daemon/PhabricatorRepositoryGraphStream.php',
3105
3107
'PhabricatorRepositoryManagementCacheWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementCacheWorkflow.php',
···
7665
7667
'PhabricatorRepositoryEngine' => 'Phobject',
7666
7668
'PhabricatorRepositoryGitCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker',
7667
7669
'PhabricatorRepositoryGitCommitMessageParserWorker' => 'PhabricatorRepositoryCommitMessageParserWorker',
7670
+
'PhabricatorRepositoryGitLFSRef' => array(
7671
+
'PhabricatorRepositoryDAO',
7672
+
'PhabricatorPolicyInterface',
7673
+
),
7674
+
'PhabricatorRepositoryGitLFSRefQuery' => 'PhabricatorCursorPagedPolicyAwareQuery',
7668
7675
'PhabricatorRepositoryGraphCache' => 'Phobject',
7669
7676
'PhabricatorRepositoryGraphStream' => 'Phobject',
7670
7677
'PhabricatorRepositoryManagementCacheWorkflow' => 'PhabricatorRepositoryManagementWorkflow',
+140
-5
src/applications/diffusion/controller/DiffusionServeController.php
+140
-5
src/applications/diffusion/controller/DiffusionServeController.php
···
9
9
private $gitLFSToken;
10
10
11
11
public function setServiceViewer(PhabricatorUser $viewer) {
12
+
$this->getRequest()->setUser($viewer);
13
+
12
14
$this->serviceViewer = $viewer;
13
15
return $this;
14
16
}
···
42
44
43
45
$content_type = $request->getHTTPHeader('Content-Type');
44
46
$user_agent = idx($_SERVER, 'HTTP_USER_AGENT');
47
+
$request_type = $request->getHTTPHeader('X-Phabricator-Request-Type');
45
48
46
49
// This may have a "charset" suffix, so only match the prefix.
47
50
$lfs_pattern = '(^application/vnd\\.git-lfs\\+json(;|\z))';
···
62
65
$vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT;
63
66
} else if (preg_match($lfs_pattern, $content_type)) {
64
67
// This is a Git LFS HTTP API request.
68
+
$vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT;
69
+
$this->isGitLFSRequest = true;
70
+
} else if ($request_type == 'git-lfs') {
71
+
// This is a Git LFS object content request.
65
72
$vcs = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT;
66
73
$this->isGitLFSRequest = true;
67
74
} else if ($request->getExists('cmd')) {
···
884
891
}
885
892
886
893
$path = $this->getGitLFSRequestPath($repository);
894
+
if ($path == 'objects/batch') {
895
+
return $this->serveGitLFSBatchRequest($repository, $viewer);
896
+
} else {
897
+
return DiffusionGitLFSResponse::newErrorResponse(
898
+
404,
899
+
pht(
900
+
'Git LFS operation "%s" is not supported by this server.',
901
+
$path));
902
+
}
903
+
}
904
+
905
+
private function serveGitLFSBatchRequest(
906
+
PhabricatorRepository $repository,
907
+
PhabricatorUser $viewer) {
908
+
909
+
$input = PhabricatorStartup::getRawInput();
910
+
$input = phutil_json_decode($input);
911
+
912
+
$operation = idx($input, 'operation');
913
+
switch ($operation) {
914
+
case 'upload':
915
+
$want_upload = true;
916
+
break;
917
+
case 'download':
918
+
$want_upload = false;
919
+
break;
920
+
default:
921
+
return DiffusionGitLFSResponse::newErrorResponse(
922
+
404,
923
+
pht(
924
+
'Git LFS batch operation "%s" is not supported by this server.',
925
+
$operation));
926
+
}
927
+
928
+
$objects = idx($input, 'objects', array());
887
929
888
-
return DiffusionGitLFSResponse::newErrorResponse(
889
-
404,
890
-
pht(
891
-
'Git LFS operation "%s" is not supported by this server.',
892
-
$path));
930
+
$hashes = array();
931
+
foreach ($objects as $object) {
932
+
$hashes[] = idx($object, 'oid');
933
+
}
934
+
935
+
if ($hashes) {
936
+
$refs = id(new PhabricatorRepositoryGitLFSRefQuery())
937
+
->setViewer($viewer)
938
+
->withRepositoryPHIDs(array($repository->getPHID()))
939
+
->withObjectHashes($hashes)
940
+
->execute();
941
+
$refs = mpull($refs, null, 'getObjectHash');
942
+
} else {
943
+
$refs = array();
944
+
}
945
+
946
+
$file_phids = mpull($refs, 'getFilePHID');
947
+
if ($file_phids) {
948
+
$files = id(new PhabricatorFileQuery())
949
+
->setViewer($viewer)
950
+
->withPHIDs(array($file_phids))
951
+
->execute();
952
+
$files = mpull($files, null, 'getPHID');
953
+
} else {
954
+
$files = array();
955
+
}
956
+
957
+
$authorization = null;
958
+
$output = array();
959
+
foreach ($objects as $object) {
960
+
$oid = idx($object, 'oid');
961
+
$size = idx($object, 'size');
962
+
$ref = idx($refs, $oid);
963
+
964
+
// NOTE: If we already have a ref for this object, we only emit a
965
+
// "download" action. The client should not upload the file again.
966
+
967
+
$actions = array();
968
+
if ($ref) {
969
+
$file = idx($files, $ref->getFilePHID());
970
+
if ($file) {
971
+
$get_uri = $file->getCDNURIWithToken();
972
+
$actions['download'] = array(
973
+
'href' => $get_uri,
974
+
);
975
+
}
976
+
} else if ($want_upload) {
977
+
if (!$authorization) {
978
+
// Here, we could reuse the existing authorization if we have one,
979
+
// but it's a little simpler to just generate a new one
980
+
// unconditionally.
981
+
$authorization = $this->newGitLFSHTTPAuthorization(
982
+
$repository,
983
+
$viewer,
984
+
$operation);
985
+
}
986
+
987
+
$put_uri = $repository->getGitLFSURI("info/lfs/upload/{$oid}");
988
+
989
+
$actions['upload'] = array(
990
+
'href' => $put_uri,
991
+
'header' => array(
992
+
'Authorization' => $authorization,
993
+
'X-Phabricator-Request-Type' => 'git-lfs',
994
+
),
995
+
);
996
+
}
997
+
998
+
$output[] = array(
999
+
'oid' => $oid,
1000
+
'size' => $size,
1001
+
'actions' => $actions,
1002
+
);
1003
+
}
1004
+
1005
+
$output = array(
1006
+
'objects' => $output,
1007
+
);
1008
+
1009
+
return id(new DiffusionGitLFSResponse())
1010
+
->setContent($output);
1011
+
}
1012
+
1013
+
private function newGitLFSHTTPAuthorization(
1014
+
PhabricatorRepository $repository,
1015
+
PhabricatorUser $viewer,
1016
+
$operation) {
1017
+
1018
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
1019
+
1020
+
$authorization = DiffusionGitLFSTemporaryTokenType::newHTTPAuthorization(
1021
+
$repository,
1022
+
$viewer,
1023
+
$operation);
1024
+
1025
+
unset($unguarded);
1026
+
1027
+
return $authorization;
893
1028
}
894
1029
895
1030
private function getGitLFSRequestPath(PhabricatorRepository $repository) {
+7
-17
src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php
+7
-17
src/applications/diffusion/gitlfs/DiffusionGitLFSAuthenticateWorkflow.php
···
83
83
// on this host, and does not require the user to have a VCS password.
84
84
85
85
$user = $this->getUser();
86
-
$headers = array();
87
86
88
-
$lfs_user = DiffusionGitLFSTemporaryTokenType::HTTP_USERNAME;
89
-
$lfs_pass = Filesystem::readRandomCharacters(32);
90
-
$lfs_hash = PhabricatorHash::digest($lfs_pass);
91
-
92
-
$ttl = PhabricatorTime::getNow() + phutil_units('1 day in seconds');
93
-
94
-
$token = id(new PhabricatorAuthTemporaryToken())
95
-
->setTokenResource($repository->getPHID())
96
-
->setTokenType(DiffusionGitLFSTemporaryTokenType::TOKENTYPE)
97
-
->setTokenCode($lfs_hash)
98
-
->setUserPHID($user->getPHID())
99
-
->setTemporaryTokenProperty('lfs.operation', $operation)
100
-
->setTokenExpires($ttl)
101
-
->save();
87
+
$authorization = DiffusionGitLFSTemporaryTokenType::newHTTPAuthorization(
88
+
$repository,
89
+
$user,
90
+
$operation);
102
91
103
-
$authorization_header = base64_encode($lfs_user.':'.$lfs_pass);
104
-
$headers['Authorization'] = 'Basic '.$authorization_header;
92
+
$headers = array(
93
+
'authorization' => $authorization,
94
+
);
105
95
106
96
$result = array(
107
97
'header' => $headers,
+24
src/applications/diffusion/gitlfs/DiffusionGitLFSTemporaryTokenType.php
+24
src/applications/diffusion/gitlfs/DiffusionGitLFSTemporaryTokenType.php
···
15
15
return pht('Git LFS Token');
16
16
}
17
17
18
+
public static function newHTTPAuthorization(
19
+
PhabricatorRepository $repository,
20
+
PhabricatorUser $viewer,
21
+
$operation) {
22
+
23
+
$lfs_user = self::HTTP_USERNAME;
24
+
$lfs_pass = Filesystem::readRandomCharacters(32);
25
+
$lfs_hash = PhabricatorHash::digest($lfs_pass);
26
+
27
+
$ttl = PhabricatorTime::getNow() + phutil_units('1 day in seconds');
28
+
29
+
$token = id(new PhabricatorAuthTemporaryToken())
30
+
->setTokenResource($repository->getPHID())
31
+
->setTokenType(self::TOKENTYPE)
32
+
->setTokenCode($lfs_hash)
33
+
->setUserPHID($viewer->getPHID())
34
+
->setTemporaryTokenProperty('lfs.operation', $operation)
35
+
->setTokenExpires($ttl)
36
+
->save();
37
+
38
+
$authorization_header = base64_encode($lfs_user.':'.$lfs_pass);
39
+
return 'Basic '.$authorization_header;
40
+
}
41
+
18
42
}
+64
src/applications/repository/query/PhabricatorRepositoryGitLFSRefQuery.php
+64
src/applications/repository/query/PhabricatorRepositoryGitLFSRefQuery.php
···
1
+
<?php
2
+
3
+
final class PhabricatorRepositoryGitLFSRefQuery
4
+
extends PhabricatorCursorPagedPolicyAwareQuery {
5
+
6
+
private $ids;
7
+
private $repositoryPHIDs;
8
+
private $objectHashes;
9
+
10
+
public function withIDs(array $ids) {
11
+
$this->ids = $ids;
12
+
return $this;
13
+
}
14
+
15
+
public function withRepositoryPHIDs(array $phids) {
16
+
$this->repositoryPHIDs = $phids;
17
+
return $this;
18
+
}
19
+
20
+
public function withObjectHashes(array $hashes) {
21
+
$this->objectHashes = $hashes;
22
+
return $this;
23
+
}
24
+
25
+
public function newResultObject() {
26
+
return new PhabricatorRepositoryGitLFSRef();
27
+
}
28
+
29
+
protected function loadPage() {
30
+
return $this->loadStandardPage($this->newResultObject());
31
+
}
32
+
33
+
protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) {
34
+
$where = parent::buildWhereClauseParts($conn);
35
+
36
+
if ($this->ids !== null) {
37
+
$where[] = qsprintf(
38
+
$conn,
39
+
'id IN (%Ld)',
40
+
$this->ids);
41
+
}
42
+
43
+
if ($this->repositoryPHIDs !== null) {
44
+
$where[] = qsprintf(
45
+
$conn,
46
+
'repositoryPHID IN (%Ls)',
47
+
$this->repositoryPHIDs);
48
+
}
49
+
50
+
if ($this->objectHashes !== null) {
51
+
$where[] = qsprintf(
52
+
$conn,
53
+
'objectHash IN (%Ls)',
54
+
$this->objectHashes);
55
+
}
56
+
57
+
return $where;
58
+
}
59
+
60
+
public function getQueryApplicationClass() {
61
+
return 'PhabricatorDiffusionApplication';
62
+
}
63
+
64
+
}
+51
src/applications/repository/storage/PhabricatorRepositoryGitLFSRef.php
+51
src/applications/repository/storage/PhabricatorRepositoryGitLFSRef.php
···
1
+
<?php
2
+
3
+
final class PhabricatorRepositoryGitLFSRef
4
+
extends PhabricatorRepositoryDAO
5
+
implements PhabricatorPolicyInterface {
6
+
7
+
protected $repositoryPHID;
8
+
protected $objectHash;
9
+
protected $byteSize;
10
+
protected $authorPHID;
11
+
protected $filePHID;
12
+
13
+
protected function getConfiguration() {
14
+
return array(
15
+
self::CONFIG_COLUMN_SCHEMA => array(
16
+
'objectHash' => 'bytes64',
17
+
'byteSize' => 'uint64',
18
+
),
19
+
self::CONFIG_KEY_SCHEMA => array(
20
+
'key_hash' => array(
21
+
'columns' => array('repositoryPHID', 'objectHash'),
22
+
'unique' => true,
23
+
),
24
+
),
25
+
) + parent::getConfiguration();
26
+
}
27
+
28
+
29
+
/* -( PhabricatorPolicyInterface )----------------------------------------- */
30
+
31
+
32
+
public function getCapabilities() {
33
+
return array(
34
+
PhabricatorPolicyCapability::CAN_VIEW,
35
+
);
36
+
}
37
+
38
+
public function getPolicy($capability) {
39
+
return PhabricatorPolicies::getMostOpenPolicy();
40
+
}
41
+
42
+
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
43
+
return false;
44
+
}
45
+
46
+
public function describeAutomaticCapability($capability) {
47
+
return null;
48
+
}
49
+
50
+
51
+
}