@recaptime-dev's working patches + fork for Phorge, a community fork of Phabricator. (Upstream dev and stable branches are at upstream/main and upstream/stable respectively.)
hq.recaptime.dev/wiki/Phorge
phorge
phabricator
Support pushing data into Git LFS
Summary:
Ref T7789. Ref T10604. This implements the `upload` action, which streams file data into Files.
This makes Git LFS actually work, at least roughly.
Test Plan:
- Tracked files in an LFS repository.
- Pushed LFS data (`git lfs track '*.png'; git add something.png; git commit -m ...; git push`).
- Pulled LFS data (`git checkout master^; rm -rf .git/lfs; git checkout master; open something.png`).
- Verified LFS refs show up in the gitlfsref table.
Reviewers: chad
Reviewed By: chad
Maniphest Tasks: T7789, T10604
Differential Revision: https://secure.phabricator.com/D15492
+129
-5
+2
src/__phutil_library_map__.php
+2
src/__phutil_library_map__.php
···
2501
2501
'PhabricatorInvalidConfigSetupCheck' => 'applications/config/check/PhabricatorInvalidConfigSetupCheck.php',
2502
2502
'PhabricatorIteratedMD5PasswordHasher' => 'infrastructure/util/password/PhabricatorIteratedMD5PasswordHasher.php',
2503
2503
'PhabricatorIteratedMD5PasswordHasherTestCase' => 'infrastructure/util/password/__tests__/PhabricatorIteratedMD5PasswordHasherTestCase.php',
2504
+
'PhabricatorIteratorFileUploadSource' => 'applications/files/uploadsource/PhabricatorIteratorFileUploadSource.php',
2504
2505
'PhabricatorJIRAAuthProvider' => 'applications/auth/provider/PhabricatorJIRAAuthProvider.php',
2505
2506
'PhabricatorJavelinLinter' => 'infrastructure/lint/linter/PhabricatorJavelinLinter.php',
2506
2507
'PhabricatorJiraIssueHasObjectEdgeType' => 'applications/doorkeeper/edge/PhabricatorJiraIssueHasObjectEdgeType.php',
···
6947
6948
'PhabricatorInvalidConfigSetupCheck' => 'PhabricatorSetupCheck',
6948
6949
'PhabricatorIteratedMD5PasswordHasher' => 'PhabricatorPasswordHasher',
6949
6950
'PhabricatorIteratedMD5PasswordHasherTestCase' => 'PhabricatorTestCase',
6951
+
'PhabricatorIteratorFileUploadSource' => 'PhabricatorFileUploadSource',
6950
6952
'PhabricatorJIRAAuthProvider' => 'PhabricatorOAuth1AuthProvider',
6951
6953
'PhabricatorJavelinLinter' => 'ArcanistLinter',
6952
6954
'PhabricatorJiraIssueHasObjectEdgeType' => 'PhabricatorEdgeType',
+99
-4
src/applications/diffusion/controller/DiffusionServeController.php
+99
-4
src/applications/diffusion/controller/DiffusionServeController.php
···
891
891
}
892
892
893
893
$path = $this->getGitLFSRequestPath($repository);
894
-
if ($path == 'objects/batch') {
894
+
$matches = null;
895
+
896
+
if (preg_match('(^upload/(.*)\z)', $path, $matches)) {
897
+
$oid = $matches[1];
898
+
return $this->serveGitLFSUploadRequest($repository, $viewer, $oid);
899
+
} else if ($path == 'objects/batch') {
895
900
return $this->serveGitLFSBatchRequest($repository, $viewer);
896
901
} else {
897
902
return DiffusionGitLFSResponse::newErrorResponse(
···
947
952
if ($file_phids) {
948
953
$files = id(new PhabricatorFileQuery())
949
954
->setViewer($viewer)
950
-
->withPHIDs(array($file_phids))
955
+
->withPHIDs($file_phids)
951
956
->execute();
952
957
$files = mpull($files, null, 'getPHID');
953
958
} else {
···
960
965
$oid = idx($object, 'oid');
961
966
$size = idx($object, 'size');
962
967
$ref = idx($refs, $oid);
968
+
$error = null;
963
969
964
970
// NOTE: If we already have a ref for this object, we only emit a
965
971
// "download" action. The client should not upload the file again.
···
968
974
if ($ref) {
969
975
$file = idx($files, $ref->getFilePHID());
970
976
if ($file) {
977
+
// Git LFS may prompt users for authentication if the action does
978
+
// not provide an "Authorization" header and does not have a query
979
+
// parameter named "token". See here for discussion:
980
+
// <https://github.com/github/git-lfs/issues/1088>
981
+
$no_authorization = 'Basic '.base64_encode('none');
982
+
971
983
$get_uri = $file->getCDNURIWithToken();
972
984
$actions['download'] = array(
973
985
'href' => $get_uri,
986
+
'header' => array(
987
+
'Authorization' => $no_authorization,
988
+
),
989
+
);
990
+
} else {
991
+
$error = array(
992
+
'code' => 404,
993
+
'message' => pht(
994
+
'Object "%s" was previously uploaded, but no longer exists '.
995
+
'on this server.',
996
+
$oid),
974
997
);
975
998
}
976
999
} else if ($want_upload) {
···
995
1018
);
996
1019
}
997
1020
998
-
$output[] = array(
1021
+
$object = array(
999
1022
'oid' => $oid,
1000
1023
'size' => $size,
1001
-
'actions' => $actions,
1002
1024
);
1025
+
1026
+
if ($actions) {
1027
+
$object['actions'] = $actions;
1028
+
}
1029
+
1030
+
if ($error) {
1031
+
$object['error'] = $error;
1032
+
}
1033
+
1034
+
$output[] = $object;
1003
1035
}
1004
1036
1005
1037
$output = array(
···
1008
1040
1009
1041
return id(new DiffusionGitLFSResponse())
1010
1042
->setContent($output);
1043
+
}
1044
+
1045
+
private function serveGitLFSUploadRequest(
1046
+
PhabricatorRepository $repository,
1047
+
PhabricatorUser $viewer,
1048
+
$oid) {
1049
+
1050
+
$ref = id(new PhabricatorRepositoryGitLFSRefQuery())
1051
+
->setViewer($viewer)
1052
+
->withRepositoryPHIDs(array($repository->getPHID()))
1053
+
->withObjectHashes(array($oid))
1054
+
->executeOne();
1055
+
if ($ref) {
1056
+
return DiffusionGitLFSResponse::newErrorResponse(
1057
+
405,
1058
+
pht(
1059
+
'Content for object "%s" is already known to this server. It can '.
1060
+
'not be uploaded again.',
1061
+
$oid));
1062
+
}
1063
+
1064
+
$request_stream = new AphrontRequestStream();
1065
+
$request_iterator = $request_stream->getIterator();
1066
+
$hashing_iterator = id(new PhutilHashingIterator($request_iterator))
1067
+
->setAlgorithm('sha256');
1068
+
1069
+
$source = id(new PhabricatorIteratorFileUploadSource())
1070
+
->setName('lfs-'.$oid)
1071
+
->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
1072
+
->setIterator($hashing_iterator);
1073
+
1074
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
1075
+
$file = $source->uploadFile();
1076
+
unset($unguarded);
1077
+
1078
+
$hash = $hashing_iterator->getHash();
1079
+
if ($hash !== $oid) {
1080
+
return DiffusionGitLFSResponse::newErrorResponse(
1081
+
400,
1082
+
pht(
1083
+
'Uploaded data is corrupt or invalid. Expected hash "%s", actual '.
1084
+
'hash "%s".',
1085
+
$oid,
1086
+
$hash));
1087
+
}
1088
+
1089
+
$ref = id(new PhabricatorRepositoryGitLFSRef())
1090
+
->setRepositoryPHID($repository->getPHID())
1091
+
->setObjectHash($hash)
1092
+
->setByteSize($file->getByteSize())
1093
+
->setAuthorPHID($viewer->getPHID())
1094
+
->setFilePHID($file->getPHID());
1095
+
1096
+
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
1097
+
// Attach the file to the repository to give users permission
1098
+
// to access it.
1099
+
$file->attachToObject($repository->getPHID());
1100
+
$ref->save();
1101
+
unset($unguarded);
1102
+
1103
+
// This is just a plain HTTP 200 with no content, which is what `git lfs`
1104
+
// expects.
1105
+
return new DiffusionGitLFSResponse();
1011
1106
}
1012
1107
1013
1108
private function newGitLFSHTTPAuthorization(
+3
-1
src/applications/files/uploadsource/PhabricatorFileUploadSource.php
+3
-1
src/applications/files/uploadsource/PhabricatorFileUploadSource.php
+25
src/applications/files/uploadsource/PhabricatorIteratorFileUploadSource.php
+25
src/applications/files/uploadsource/PhabricatorIteratorFileUploadSource.php
···
1
+
<?php
2
+
3
+
final class PhabricatorIteratorFileUploadSource
4
+
extends PhabricatorFileUploadSource {
5
+
6
+
private $iterator;
7
+
8
+
public function setIterator(Iterator $iterator) {
9
+
$this->iterator = $iterator;
10
+
return $this;
11
+
}
12
+
13
+
public function getIterator() {
14
+
return $this->iterator;
15
+
}
16
+
17
+
protected function newDataIterator() {
18
+
return $this->getIterator();
19
+
}
20
+
21
+
protected function getDataLength() {
22
+
return null;
23
+
}
24
+
25
+
}