Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

appview,knotmirror: use knotmirror to read the repository

Underlying types except the interface hasn't changed much.
Removed `xrpcclient.HandleXrpcErr()` call as appview always expect
knotmirror with compatible API.

Signed-off-by: Seongmin Lee <git@boltless.me>

authored by

Seongmin Lee and committed by tangled.org 35ceae65 72c1b45b

+1082 -263
+21 -16
appview/config/config.go
··· 46 46 PLCURL string `env:"URL, default=https://plc.directory"` 47 47 } 48 48 49 + type KnotMirrorConfig struct { 50 + Url string `env:"URL, required"` 51 + } 52 + 49 53 type JetstreamConfig struct { 50 54 Endpoint string `env:"ENDPOINT, default=wss://jetstream1.us-east.bsky.network/subscribe"` 51 55 } ··· 154 150 } 155 151 156 152 type Config struct { 157 - Core CoreConfig `env:",prefix=TANGLED_"` 158 - Jetstream JetstreamConfig `env:",prefix=TANGLED_JETSTREAM_"` 159 - Knotstream ConsumerConfig `env:",prefix=TANGLED_KNOTSTREAM_"` 160 - Spindlestream ConsumerConfig `env:",prefix=TANGLED_SPINDLESTREAM_"` 161 - Resend ResendConfig `env:",prefix=TANGLED_RESEND_"` 162 - Posthog PosthogConfig `env:",prefix=TANGLED_POSTHOG_"` 163 - Camo CamoConfig `env:",prefix=TANGLED_CAMO_"` 164 - Avatar AvatarConfig `env:",prefix=TANGLED_AVATAR_"` 165 - OAuth OAuthConfig `env:",prefix=TANGLED_OAUTH_"` 166 - Redis RedisConfig `env:",prefix=TANGLED_REDIS_"` 167 - Plc PlcConfig `env:",prefix=TANGLED_PLC_"` 168 - Pds PdsConfig `env:",prefix=TANGLED_PDS_"` 169 - Cloudflare Cloudflare `env:",prefix=TANGLED_CLOUDFLARE_"` 170 - Label LabelConfig `env:",prefix=TANGLED_LABEL_"` 171 - Bluesky BlueskyConfig `env:",prefix=TANGLED_BLUESKY_"` 172 - Sites SitesConfig `env:",prefix=TANGLED_SITES_"` 153 + Core CoreConfig `env:",prefix=TANGLED_"` 154 + Jetstream JetstreamConfig `env:",prefix=TANGLED_JETSTREAM_"` 155 + Knotstream ConsumerConfig `env:",prefix=TANGLED_KNOTSTREAM_"` 156 + Spindlestream ConsumerConfig `env:",prefix=TANGLED_SPINDLESTREAM_"` 157 + Resend ResendConfig `env:",prefix=TANGLED_RESEND_"` 158 + Posthog PosthogConfig `env:",prefix=TANGLED_POSTHOG_"` 159 + Camo CamoConfig `env:",prefix=TANGLED_CAMO_"` 160 + Avatar AvatarConfig `env:",prefix=TANGLED_AVATAR_"` 161 + OAuth OAuthConfig `env:",prefix=TANGLED_OAUTH_"` 162 + Redis RedisConfig `env:",prefix=TANGLED_REDIS_"` 163 + Plc PlcConfig `env:",prefix=TANGLED_PLC_"` 164 + Pds PdsConfig `env:",prefix=TANGLED_PDS_"` 165 + Cloudflare Cloudflare `env:",prefix=TANGLED_CLOUDFLARE_"` 166 + Label LabelConfig `env:",prefix=TANGLED_LABEL_"` 167 + Bluesky BlueskyConfig `env:",prefix=TANGLED_BLUESKY_"` 168 + Sites SitesConfig `env:",prefix=TANGLED_SITES_"` 169 + KnotMirror KnotMirrorConfig `env:",prefix=TANGLED_KNOTMIRROR_"` 173 170 } 174 171 175 172 func LoadConfig(ctx context.Context) (*Config, error) {
+16 -82
appview/pulls/pulls.go
··· 411 411 return nil 412 412 } 413 413 414 - scheme := "http" 415 - if !s.config.Core.Dev { 416 - scheme = "https" 417 - } 418 - host := fmt.Sprintf("%s://%s", scheme, repo.Knot) 419 - xrpcc := &indigoxrpc.Client{ 420 - Host: host, 421 - } 422 - 423 - resp, err := tangled.RepoBranch(r.Context(), xrpcc, branch, fmt.Sprintf("%s/%s", repo.Did, repo.Name)) 414 + xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} 415 + resp, err := tangled.GitTempGetBranch(r.Context(), xrpcc, branch, repo.RepoAt().String()) 424 416 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 425 417 return nil 426 418 } ··· 428 436 return pages.Unknown 429 437 } 430 438 431 - var knot, ownerDid, repoName string 432 - 439 + var sourceRepo syntax.ATURI 433 440 if pull.PullSource.RepoAt != nil { 434 441 // fork-based pulls 435 - sourceRepo, err := db.GetRepoByAtUri(s.db, pull.PullSource.RepoAt.String()) 436 - if err != nil { 437 - log.Println("failed to get source repo", err) 438 - return pages.Unknown 439 - } 440 - 441 - knot = sourceRepo.Knot 442 - ownerDid = sourceRepo.Did 443 - repoName = sourceRepo.Name 442 + sourceRepo = *pull.PullSource.RepoAt 444 443 } else { 445 444 // pulls within the same repo 446 - knot = repo.Knot 447 - ownerDid = repo.Did 448 - repoName = repo.Name 445 + sourceRepo = repo.RepoAt() 449 446 } 450 447 451 - scheme := "http" 452 - if !s.config.Core.Dev { 453 - scheme = "https" 454 - } 455 - host := fmt.Sprintf("%s://%s", scheme, knot) 456 - xrpcc := &indigoxrpc.Client{ 457 - Host: host, 458 - } 459 - 460 - didSlashName := fmt.Sprintf("%s/%s", ownerDid, repoName) 461 - branchResp, err := tangled.RepoBranch(r.Context(), xrpcc, pull.PullSource.Branch, didSlashName) 448 + xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} 449 + branchResp, err := tangled.GitTempGetBranch(r.Context(), xrpcc, pull.PullSource.Branch, sourceRepo.String()) 462 450 if err != nil { 463 451 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 464 452 log.Println("failed to call XRPC repo.branches", xrpcerr) ··· 876 904 877 905 switch r.Method { 878 906 case http.MethodGet: 879 - scheme := "http" 880 - if !s.config.Core.Dev { 881 - scheme = "https" 882 - } 883 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 884 - xrpcc := &indigoxrpc.Client{ 885 - Host: host, 886 - } 907 + xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} 887 908 888 - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) 889 - xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) 909 + xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) 890 910 if err != nil { 891 911 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 892 912 log.Println("failed to call XRPC repo.branches", xrpcerr) ··· 1499 1535 return 1500 1536 } 1501 1537 1502 - scheme := "http" 1503 - if !s.config.Core.Dev { 1504 - scheme = "https" 1505 - } 1506 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 1507 - xrpcc := &indigoxrpc.Client{ 1508 - Host: host, 1509 - } 1538 + xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} 1510 1539 1511 - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) 1512 - xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) 1540 + xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) 1513 1541 if err != nil { 1514 - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 1515 - log.Println("failed to call XRPC repo.branches", xrpcerr) 1516 - s.pages.Error503(w) 1517 - return 1518 - } 1519 1542 log.Println("failed to fetch branches", err) 1543 + s.pages.Error503(w) 1520 1544 return 1521 1545 } 1522 1546 ··· 1559 1607 return 1560 1608 } 1561 1609 1610 + xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} 1611 + 1562 1612 forkVal := r.URL.Query().Get("fork") 1563 1613 repoString := strings.SplitN(forkVal, "/", 2) 1564 1614 forkOwnerDid := repoString[0] ··· 1576 1622 return 1577 1623 } 1578 1624 1579 - sourceScheme := "http" 1580 - if !s.config.Core.Dev { 1581 - sourceScheme = "https" 1582 - } 1583 - sourceHost := fmt.Sprintf("%s://%s", sourceScheme, repo.Knot) 1584 - sourceXrpcc := &indigoxrpc.Client{ 1585 - Host: sourceHost, 1586 - } 1587 - 1588 - sourceRepo := fmt.Sprintf("%s/%s", forkOwnerDid, repo.Name) 1589 - sourceXrpcBytes, err := tangled.RepoBranches(r.Context(), sourceXrpcc, "", 0, sourceRepo) 1625 + sourceXrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, repo.RepoAt().String()) 1590 1626 if err != nil { 1591 1627 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 1592 1628 log.Println("failed to call XRPC repo.branches for source", xrpcerr) ··· 1595 1651 return 1596 1652 } 1597 1653 1598 - targetScheme := "http" 1599 - if !s.config.Core.Dev { 1600 - targetScheme = "https" 1601 - } 1602 - targetHost := fmt.Sprintf("%s://%s", targetScheme, f.Knot) 1603 - targetXrpcc := &indigoxrpc.Client{ 1604 - Host: targetHost, 1605 - } 1606 - 1607 - targetRepo := fmt.Sprintf("%s/%s", f.Did, f.Name) 1608 - targetXrpcBytes, err := tangled.RepoBranches(r.Context(), targetXrpcc, "", 0, targetRepo) 1654 + targetXrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) 1609 1655 if err != nil { 1610 1656 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 1611 1657 log.Println("failed to call XRPC repo.branches for target", xrpcerr)
+9 -19
appview/repo/archive.go
··· 8 8 "strings" 9 9 10 10 "github.com/go-chi/chi/v5" 11 + "tangled.org/core/api/tangled" 11 12 ) 12 13 13 14 func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) { ··· 21 20 l.Error("failed to get repo and knot", "err", err) 22 21 return 23 22 } 24 - scheme := "http" 25 - if !rp.config.Core.Dev { 26 - scheme = "https" 27 - } 28 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 29 - didSlashRepo := f.DidSlashRepo() 30 23 31 24 // build the xrpc url 32 - u, err := url.Parse(host) 33 - if err != nil { 34 - l.Error("failed to parse host URL", "err", err) 35 - rp.pages.Error503(w) 36 - return 37 - } 38 - 39 - u.Path = "/xrpc/sh.tangled.repo.archive" 40 25 query := url.Values{} 26 + query.Set("repo", f.RepoAt().String()) 27 + query.Set("ref", ref) 41 28 query.Set("format", "tar.gz") 42 29 query.Set("prefix", r.URL.Query().Get("prefix")) 43 - query.Set("ref", ref) 44 - query.Set("repo", didSlashRepo) 45 - u.RawQuery = query.Encode() 46 - 47 - xrpcURL := u.String() 30 + xrpcURL := fmt.Sprintf( 31 + "%s/xrpc/%s?%s", 32 + rp.config.KnotMirror.Url, 33 + tangled.GitTempGetArchiveNSID, 34 + query.Encode(), 35 + ) 48 36 49 37 // make the get request 50 38 resp, err := http.Get(xrpcURL)
+2 -10
appview/repo/artifact.go
··· 313 313 return nil, err 314 314 } 315 315 316 - scheme := "http" 317 - if !rp.config.Core.Dev { 318 - scheme = "https" 319 - } 320 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 321 - xrpcc := &indigoxrpc.Client{ 322 - Host: host, 323 - } 316 + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} 324 317 325 - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) 326 - xrpcBytes, err := tangled.RepoTags(ctx, xrpcc, "", 0, repo) 318 + xrpcBytes, err := tangled.GitTempListTags(ctx, xrpcc, "", 0, f.RepoAt().String()) 327 319 if err != nil { 328 320 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 329 321 l.Error("failed to call XRPC repo.tags", "err", xrpcerr)
+5 -12
appview/repo/branches.go
··· 21 21 l.Error("failed to get repo and knot", "err", err) 22 22 return 23 23 } 24 - scheme := "http" 25 - if !rp.config.Core.Dev { 26 - scheme = "https" 27 - } 28 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 29 - xrpcc := &indigoxrpc.Client{ 30 - Host: host, 31 - } 32 - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) 33 - xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) 34 - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 35 - l.Error("failed to call XRPC repo.branches", "err", xrpcerr) 24 + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} 25 + 26 + xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) 27 + if err != nil { 28 + l.Error("failed to call XRPC repo.branches", "err", err) 36 29 rp.pages.Error503(w) 37 30 return 38 31 }
+3 -11
appview/repo/compare.go
··· 27 27 return 28 28 } 29 29 30 - scheme := "http" 31 - if !rp.config.Core.Dev { 32 - scheme = "https" 33 - } 34 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 35 - xrpcc := &indigoxrpc.Client{ 36 - Host: host, 37 - } 30 + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} 38 31 39 - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) 40 - branchBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) 32 + branchBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) 41 33 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 42 34 l.Error("failed to call XRPC repo.branches", "err", xrpcerr) 43 35 rp.pages.Error503(w) ··· 66 74 head = queryHead 67 75 } 68 76 69 - tagBytes, err := tangled.RepoTags(r.Context(), xrpcc, "", 0, repo) 77 + tagBytes, err := tangled.GitTempListTags(r.Context(), xrpcc, "", 0, f.RepoAt().String()) 70 78 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 71 79 l.Error("failed to call XRPC repo.tags", "err", xrpcerr) 72 80 rp.pages.Error503(w)
+27 -52
appview/repo/index.go
··· 22 22 "tangled.org/core/appview/db" 23 23 "tangled.org/core/appview/models" 24 24 "tangled.org/core/appview/pages" 25 - "tangled.org/core/appview/xrpcclient" 26 25 "tangled.org/core/orm" 27 26 "tangled.org/core/types" 28 27 ··· 41 42 return 42 43 } 43 44 44 - scheme := "http" 45 - if !rp.config.Core.Dev { 46 - scheme = "https" 47 - } 48 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 49 - xrpcc := &indigoxrpc.Client{ 50 - Host: host, 51 - } 52 - 53 45 user := rp.oauth.GetMultiAccountUser(r) 54 46 55 47 // Build index response from multiple XRPC calls 56 - result, err := rp.buildIndexResponse(r.Context(), xrpcc, f, ref) 57 - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 58 - if errors.Is(xrpcerr, xrpcclient.ErrXrpcUnsupported) { 59 - l.Error("failed to call XRPC repo.index", "err", err) 60 - rp.pages.RepoIndexPage(w, pages.RepoIndexParams{ 61 - LoggedInUser: user, 62 - NeedsKnotUpgrade: true, 63 - RepoInfo: rp.repoResolver.GetRepoInfo(r, user), 64 - }) 65 - return 66 - } else { 67 - l.Error("failed to build index response", "err", err) 68 - rp.pages.RepoIndexPage(w, pages.RepoIndexParams{ 69 - LoggedInUser: user, 70 - KnotUnreachable: true, 71 - RepoInfo: rp.repoResolver.GetRepoInfo(r, user), 72 - }) 73 - return 74 - } 48 + result, err := rp.buildIndexResponse(r.Context(), f, ref) 49 + if err != nil { 50 + l.Error("failed to build index response", "err", err) 51 + rp.pages.RepoIndexPage(w, pages.RepoIndexParams{ 52 + LoggedInUser: user, 53 + KnotUnreachable: true, 54 + RepoInfo: rp.repoResolver.GetRepoInfo(r, user), 55 + }) 56 + return 75 57 } 76 58 77 59 tagMap := make(map[string][]string) ··· 113 133 } 114 134 115 135 // TODO: a bit dirty 116 - languageInfo, err := rp.getLanguageInfo(r.Context(), l, f, xrpcc, result.Ref, ref == "") 136 + languageInfo, err := rp.getLanguageInfo(r.Context(), l, f, result.Ref, ref == "") 117 137 if err != nil { 118 138 l.Warn("failed to compute language percentages", "err", err) 119 139 // non-fatal ··· 149 169 ctx context.Context, 150 170 l *slog.Logger, 151 171 repo *models.Repo, 152 - xrpcc *indigoxrpc.Client, 153 172 currentRef string, 154 173 isDefaultRef bool, 155 174 ) ([]types.RepoLanguageDetails, error) { ··· 161 182 162 183 if err != nil || langs == nil { 163 184 // non-fatal, fetch langs from ks via XRPC 164 - didSlashRepo := fmt.Sprintf("%s/%s", repo.Did, repo.Name) 165 - ls, err := tangled.RepoLanguages(ctx, xrpcc, currentRef, didSlashRepo) 185 + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} 186 + ls, err := tangled.GitTempListLanguages(ctx, xrpcc, currentRef, repo.RepoAt().String()) 166 187 if err != nil { 167 - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 168 - l.Error("failed to call XRPC repo.languages", "err", xrpcerr) 169 - return nil, xrpcerr 170 - } 171 - return nil, err 188 + return nil, fmt.Errorf("calling knotmirror git.listLanguages: %w", err) 172 189 } 173 190 174 191 if ls == nil || ls.Languages == nil { ··· 233 258 } 234 259 235 260 // buildIndexResponse creates a RepoIndexResponse by combining multiple xrpc calls in parallel 236 - func (rp *Repo) buildIndexResponse(ctx context.Context, xrpcc *indigoxrpc.Client, repo *models.Repo, ref string) (*types.RepoIndexResponse, error) { 237 - didSlashRepo := fmt.Sprintf("%s/%s", repo.Did, repo.Name) 261 + func (rp *Repo) buildIndexResponse(ctx context.Context, repo *models.Repo, ref string) (*types.RepoIndexResponse, error) { 262 + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} 238 263 239 264 // first get branches to determine the ref if not specified 240 - branchesBytes, err := tangled.RepoBranches(ctx, xrpcc, "", 0, didSlashRepo) 265 + branchesBytes, err := tangled.GitTempListBranches(ctx, xrpcc, "", 0, repo.RepoAt().String()) 241 266 if err != nil { 242 - return nil, fmt.Errorf("failed to call repoBranches: %w", err) 267 + return nil, fmt.Errorf("calling knotmirror git.listBranches: %w", err) 243 268 } 244 269 245 270 var branchesResp types.RepoBranchesResponse ··· 271 296 272 297 var ( 273 298 tagsResp types.RepoTagsResponse 274 - treeResp *tangled.RepoTree_Output 299 + treeResp *tangled.GitTempGetTree_Output 275 300 logResp types.RepoLogResponse 276 301 readmeContent string 277 302 readmeFileName string ··· 279 304 280 305 // tags 281 306 wg.Go(func() { 282 - tagsBytes, err := tangled.RepoTags(ctx, xrpcc, "", 0, didSlashRepo) 307 + tagsBytes, err := tangled.GitTempListTags(ctx, xrpcc, "", 0, repo.RepoAt().String()) 283 308 if err != nil { 284 - errs = errors.Join(errs, fmt.Errorf("failed to call repoTags: %w", err)) 309 + errs = errors.Join(errs, fmt.Errorf("failed to call git.ListTags: %w", err)) 285 310 return 286 311 } 287 312 288 313 if err := json.Unmarshal(tagsBytes, &tagsResp); err != nil { 289 - errs = errors.Join(errs, fmt.Errorf("failed to unmarshal repoTags: %w", err)) 314 + errs = errors.Join(errs, fmt.Errorf("failed to unmarshal git.ListTags: %w", err)) 290 315 } 291 316 }) 292 317 293 318 // tree/files 294 319 wg.Go(func() { 295 - resp, err := tangled.RepoTree(ctx, xrpcc, "", ref, didSlashRepo) 320 + resp, err := tangled.GitTempGetTree(ctx, xrpcc, "", ref, repo.RepoAt().String()) 296 321 if err != nil { 297 - errs = errors.Join(errs, fmt.Errorf("failed to call repoTree: %w", err)) 322 + errs = errors.Join(errs, fmt.Errorf("failed to call git.GetTree: %w", err)) 298 323 return 299 324 } 300 325 treeResp = resp ··· 302 327 303 328 // commits 304 329 wg.Go(func() { 305 - logBytes, err := tangled.RepoLog(ctx, xrpcc, "", 50, "", ref, didSlashRepo) 330 + logBytes, err := tangled.GitTempListCommits(ctx, xrpcc, "", 50, ref, repo.RepoAt().String()) 306 331 if err != nil { 307 - errs = errors.Join(errs, fmt.Errorf("failed to call repoLog: %w", err)) 332 + errs = errors.Join(errs, fmt.Errorf("failed to call git.ListCommits: %w", err)) 308 333 return 309 334 } 310 335 311 336 if err := json.Unmarshal(logBytes, &logResp); err != nil { 312 - errs = errors.Join(errs, fmt.Errorf("failed to unmarshal repoLog: %w", err)) 337 + errs = errors.Join(errs, fmt.Errorf("failed to unmarshal git.ListCommits: %w", err)) 313 338 } 314 339 }) 315 340 ··· 351 376 Readme: readmeContent, 352 377 ReadmeFileName: readmeFileName, 353 378 Commits: logResp.Commits, 354 - Description: logResp.Description, 379 + Description: "", 355 380 Files: files, 356 381 Branches: branchesResp.Branches, 357 382 Tags: tagsResp.Tags,
+10 -18
appview/repo/log.go
··· 40 40 ref := chi.URLParam(r, "ref") 41 41 ref, _ = url.PathUnescape(ref) 42 42 43 - scheme := "http" 44 - if !rp.config.Core.Dev { 45 - scheme = "https" 46 - } 47 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 48 - xrpcc := &indigoxrpc.Client{ 49 - Host: host, 50 - } 43 + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} 51 44 52 45 limit := int64(60) 53 46 cursor := "" ··· 50 57 cursor = strconv.Itoa(offset) 51 58 } 52 59 53 - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) 54 - xrpcBytes, err := tangled.RepoLog(r.Context(), xrpcc, cursor, limit, "", ref, repo) 55 - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 56 - l.Error("failed to call XRPC repo.log", "err", xrpcerr) 60 + xrpcBytes, err := tangled.GitTempListCommits(r.Context(), xrpcc, cursor, limit, ref, f.RepoAt().String()) 61 + if err != nil { 62 + l.Error("failed to call XRPC repo.log", "err", err) 57 63 rp.pages.Error503(w) 58 64 return 59 65 } ··· 64 72 return 65 73 } 66 74 67 - tagBytes, err := tangled.RepoTags(r.Context(), xrpcc, "", 0, repo) 68 - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 69 - l.Error("failed to call XRPC repo.tags", "err", xrpcerr) 75 + tagBytes, err := tangled.GitTempListTags(r.Context(), xrpcc, "", 0, f.RepoAt().String()) 76 + if err != nil { 77 + l.Error("failed to call XRPC repo.tags", "err", err) 70 78 rp.pages.Error503(w) 71 79 return 72 80 } ··· 85 93 } 86 94 } 87 95 88 - branchBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) 89 - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 90 - l.Error("failed to call XRPC repo.branches", "err", xrpcerr) 96 + branchBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) 97 + if err != nil { 98 + l.Error("failed to call XRPC repo.branches", "err", err) 91 99 rp.pages.Error503(w) 92 100 return 93 101 }
+2 -10
appview/repo/settings.go
··· 386 386 f, err := rp.repoResolver.Resolve(r) 387 387 user := rp.oauth.GetMultiAccountUser(r) 388 388 389 - scheme := "http" 390 - if !rp.config.Core.Dev { 391 - scheme = "https" 392 - } 393 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 394 - xrpcc := &indigoxrpc.Client{ 395 - Host: host, 396 - } 389 + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} 397 390 398 - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) 399 - xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) 391 + xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) 400 392 var result types.RepoBranchesResponse 401 393 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 402 394 l.Error("failed to call XRPC repo.branches", "err", xrpcerr)
+8 -23
appview/repo/tags.go
··· 27 27 l.Error("failed to get repo and knot", "err", err) 28 28 return 29 29 } 30 - scheme := "http" 31 - if !rp.config.Core.Dev { 32 - scheme = "https" 33 - } 34 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 35 - xrpcc := &indigoxrpc.Client{ 36 - Host: host, 37 - } 38 - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) 39 - xrpcBytes, err := tangled.RepoTags(r.Context(), xrpcc, "", 0, repo) 40 - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 41 - l.Error("failed to call XRPC repo.tags", "err", xrpcerr) 30 + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} 31 + xrpcBytes, err := tangled.GitTempListTags(r.Context(), xrpcc, "", 0, f.RepoAt().String()) 32 + if err != nil { 33 + l.Error("failed to call XRPC repo.tags", "err", err) 42 34 rp.pages.Error503(w) 43 35 return 44 36 } ··· 82 90 l.Error("failed to get repo and knot", "err", err) 83 91 return 84 92 } 85 - scheme := "http" 86 - if !rp.config.Core.Dev { 87 - scheme = "https" 88 - } 89 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 90 - xrpcc := &indigoxrpc.Client{ 91 - Host: host, 92 - } 93 - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) 94 93 tag := chi.URLParam(r, "tag") 95 94 96 - xrpcBytes, err := tangled.RepoTag(r.Context(), xrpcc, repo, tag) 95 + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} 96 + 97 + xrpcBytes, err := tangled.GitTempGetTag(r.Context(), xrpcc, f.RepoAt().String(), tag) 97 98 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 98 99 // if we don't match an existing tag, and the tag we're trying 99 100 // to match is "latest", resolve to the most recent tag 100 101 if tag == "latest" { 101 - tagsBytes, err := tangled.RepoTags(r.Context(), xrpcc, "", 1, repo) 102 + tagsBytes, err := tangled.GitTempListTags(r.Context(), xrpcc, "", 1, f.RepoAt().String()) 102 103 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 103 104 l.Error("failed to call XRPC repo.tags for latest", "err", xrpcerr) 104 105 rp.pages.Error503(w)
+3 -10
appview/repo/tree.go
··· 33 33 treePath := chi.URLParam(r, "*") 34 34 treePath, _ = url.PathUnescape(treePath) 35 35 treePath = strings.TrimSuffix(treePath, "/") 36 - scheme := "http" 37 - if !rp.config.Core.Dev { 38 - scheme = "https" 39 - } 40 - host := fmt.Sprintf("%s://%s", scheme, f.Knot) 41 - xrpcc := &indigoxrpc.Client{ 42 - Host: host, 43 - } 44 - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) 45 - xrpcResp, err := tangled.RepoTree(r.Context(), xrpcc, treePath, ref, repo) 36 + 37 + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} 38 + xrpcResp, err := tangled.GitTempGetTree(r.Context(), xrpcc, treePath, ref, f.RepoAt().String()) 46 39 if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { 47 40 l.Error("failed to call XRPC repo.tree", "err", xrpcerr) 48 41 rp.pages.Error503(w)
+11
knotmirror/config/config.go
··· 8 8 ) 9 9 10 10 type Config struct { 11 + PlcUrl string `env:"MIRROR_PLC_URL, default=https://plc.directory"` 11 12 TapUrl string `env:"MIRROR_TAP_URL, default=http://localhost:2480"` 12 13 DbUrl string `env:"MIRROR_DB_URL, required"` 13 14 KnotUseSSL bool `env:"MIRROR_KNOT_USE_SSL, default=false"` // use SSL for Knot when not scheme is not specified ··· 17 16 GitRepoFetchTimeout time.Duration `env:"MIRROR_GIT_FETCH_TIMEOUT, default=600s"` 18 17 ResyncParallelism int `env:"MIRROR_RESYNC_PARALLELISM, default=5"` 19 18 Slurper SlurperConfig `env:",prefix=MIRROR_SLURPER_"` 19 + UseSSL bool `env:"MIRROR_USE_SSL, default=false"` 20 + Hostname string `env:"MIRROR_HOSTNAME, required"` 21 + Listen string `env:"MIRROR_LISTEN, default=:7000"` 20 22 MetricsListen string `env:"MIRROR_METRICS_LISTEN, default=127.0.0.1:7100"` 21 23 AdminListen string `env:"MIRROR_ADMIN_LISTEN, default=127.0.0.1:7200"` 24 + } 25 + 26 + func (c *Config) BaseUrl() string { 27 + if c.UseSSL { 28 + return "https://" + c.Hostname 29 + } 30 + return "http://" + c.Hostname 22 31 } 23 32 24 33 type SlurperConfig struct {
+21
knotmirror/knotmirror.go
··· 7 7 _ "net/http/pprof" 8 8 "time" 9 9 10 + "github.com/go-chi/chi/v5" 10 11 "github.com/prometheus/client_golang/prometheus/promhttp" 12 + "tangled.org/core/idresolver" 11 13 "tangled.org/core/knotmirror/config" 12 14 "tangled.org/core/knotmirror/db" 13 15 "tangled.org/core/knotmirror/knotstream" 14 16 "tangled.org/core/knotmirror/models" 17 + "tangled.org/core/knotmirror/xrpc" 15 18 "tangled.org/core/log" 16 19 ) 17 20 ··· 29 26 if err != nil { 30 27 return fmt.Errorf("initializing db: %w", err) 31 28 } 29 + 30 + resolver := idresolver.DefaultResolver(cfg.PlcUrl) 32 31 33 32 // NOTE: using plain git-cli for clone/fetch as go-git is too memory-intensive. 34 33 gitm := NewCliGitMirrorManager(cfg.GitRepoBasePath, cfg.KnotUseSSL) ··· 49 44 } 50 45 logger.Info(fmt.Sprintf("clearing resyning states: %d records updated", rows)) 51 46 47 + xrpc := xrpc.New(logger, cfg, db, resolver) 52 48 knotstream := knotstream.NewKnotStream(logger, db, cfg) 53 49 crawler := NewCrawler(logger, db) 54 50 resyncer := NewResyncer(logger, db, gitm, cfg) ··· 58 52 // maintain repository list with tap 59 53 // NOTE: this can be removed once we introduce did-for-repo because then we can just listen to KnotStream for #identity events. 60 54 tap := NewTapClient(logger, cfg, db, gitm, knotstream) 55 + 56 + // start http server 57 + go func() { 58 + logger.Info("starting http server", "addr", cfg.Listen) 59 + 60 + mux := chi.NewRouter() 61 + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 62 + w.Write([]byte("Welcome to a knotmirror server.\n")) 63 + }) 64 + mux.Mount("/xrpc", xrpc.Router()) 65 + 66 + if err := http.ListenAndServe(cfg.Listen, mux); err != nil { 67 + logger.Error("xrpc server failed", "error", err) 68 + } 69 + }() 61 70 62 71 // start metrics endpoint 63 72 go func() {
+106
knotmirror/xrpc/git_getArchive.go
··· 1 + package xrpc 2 + 3 + import ( 4 + "compress/gzip" 5 + "fmt" 6 + "net/http" 7 + "net/url" 8 + "strings" 9 + 10 + "github.com/bluesky-social/indigo/atproto/atclient" 11 + "github.com/bluesky-social/indigo/atproto/syntax" 12 + "github.com/go-git/go-git/v5/plumbing" 13 + "tangled.org/core/api/tangled" 14 + "tangled.org/core/knotmirror/db" 15 + "tangled.org/core/knotserver/git" 16 + ) 17 + 18 + func (x *Xrpc) GetArchive(w http.ResponseWriter, r *http.Request) { 19 + var ( 20 + repoQuery = r.URL.Query().Get("repo") 21 + ref = r.URL.Query().Get("ref") 22 + format = r.URL.Query().Get("format") 23 + prefix = r.URL.Query().Get("prefix") 24 + ) 25 + 26 + repo, err := syntax.ParseATURI(repoQuery) 27 + if err != nil || repo.RecordKey() == "" { 28 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) 29 + return 30 + } 31 + 32 + if format != "tar.gz" { 33 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "only tar.gz format is supported"}) 34 + return 35 + } 36 + if format == "" { 37 + format = "tar.gz" 38 + } 39 + 40 + l := x.logger.With("repo", repo, "ref", ref, "format", format, "prefix", prefix) 41 + ctx := r.Context() 42 + 43 + repoPath, err := x.makeRepoPath(ctx, repo) 44 + if err != nil { 45 + l.Error("failed to resolve repo at-uri", "err", err) 46 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to resolve repo"}) 47 + return 48 + } 49 + 50 + gr, err := git.Open(repoPath, ref) 51 + if err != nil { 52 + l.Error("failed to open git repo", "err", err) 53 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to open git repo"}) 54 + return 55 + } 56 + 57 + repoName, err := func() (string, error) { 58 + r, err := db.GetRepoByAtUri(ctx, x.db, repo) 59 + if err != nil { 60 + return "", err 61 + } 62 + if r == nil { 63 + return "", fmt.Errorf("repo not found: %s", repo) 64 + } 65 + return r.Name, nil 66 + }() 67 + if err != nil { 68 + l.Error("failed to get repo name", "err", err) 69 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to retrieve repo name"}) 70 + return 71 + } 72 + 73 + safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") 74 + immutableLink := func() string { 75 + params := url.Values{} 76 + params.Set("repo", repo.String()) 77 + params.Set("ref", gr.Hash().String()) 78 + params.Set("format", format) 79 + params.Set("prefix", prefix) 80 + return fmt.Sprintf("%s/xrpc/%s?%s", x.cfg.BaseUrl(), tangled.GitTempGetArchiveNSID, params.Encode()) 81 + }() 82 + 83 + filename := fmt.Sprintf("%s-%s.tar.gz", repoName, safeRefFilename) 84 + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) 85 + w.Header().Set("Content-Type", "application/gzip") 86 + w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink)) 87 + 88 + gw := gzip.NewWriter(w) 89 + defer gw.Close() 90 + 91 + if err := gr.WriteTar(gw, prefix); err != nil { 92 + // once we start writing to the body we can't report error anymore 93 + // so we are only left with logging the error 94 + l.Error("writing tar file", "err", err.Error()) 95 + w.WriteHeader(http.StatusInternalServerError) 96 + return 97 + } 98 + 99 + if err := gw.Flush(); err != nil { 100 + // once we start writing to the body we can't report error anymore 101 + // so we are only left with logging the error 102 + l.Error("flushing", "err", err.Error()) 103 + w.WriteHeader(http.StatusInternalServerError) 104 + return 105 + } 106 + }
+86
knotmirror/xrpc/git_getBlob.go
··· 1 + package xrpc 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "io" 7 + "net/http" 8 + "slices" 9 + 10 + "github.com/bluesky-social/indigo/atproto/atclient" 11 + "github.com/bluesky-social/indigo/atproto/syntax" 12 + "github.com/go-git/go-git/v5/plumbing/object" 13 + "tangled.org/core/knotserver/git" 14 + ) 15 + 16 + func (x *Xrpc) GetBlob(w http.ResponseWriter, r *http.Request) { 17 + var ( 18 + repoQuery = r.URL.Query().Get("repo") 19 + ref = r.URL.Query().Get("ref") // ref can be empty (git.Open handles this) 20 + path = r.URL.Query().Get("path") 21 + ) 22 + 23 + repo, err := syntax.ParseATURI(repoQuery) 24 + if err != nil || repo.RecordKey() == "" { 25 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) 26 + return 27 + } 28 + 29 + l := x.logger.With("repo", repo, "ref", ref, "path", path) 30 + 31 + if path == "" { 32 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing path parameter"}) 33 + return 34 + } 35 + 36 + file, err := x.getFile(r.Context(), repo, ref, path) 37 + if err != nil { 38 + // TODO: better error return 39 + l.Error("failed to get blob", "err", err) 40 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get blob"}) 41 + return 42 + } 43 + 44 + reader, err := file.Reader() 45 + if err != nil { 46 + l.Error("failed to read blob", "err", err) 47 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to read the blob"}) 48 + return 49 + } 50 + defer reader.Close() 51 + 52 + w.Header().Set("Content-Type", "application/octet-stream") 53 + if _, err := io.Copy(w, reader); err != nil { 54 + l.Error("failed to serve the blob", "err", err) 55 + } 56 + } 57 + 58 + func (x *Xrpc) getFile(ctx context.Context, repo syntax.ATURI, ref, path string) (*object.File, error) { 59 + repoPath, err := x.makeRepoPath(ctx, repo) 60 + if err != nil { 61 + return nil, fmt.Errorf("resolving repo at-uri: %w", err) 62 + } 63 + 64 + gr, err := git.Open(repoPath, ref) 65 + if err != nil { 66 + return nil, fmt.Errorf("opening git repo: %w", err) 67 + } 68 + 69 + return gr.File(path) 70 + } 71 + 72 + var textualMimeTypes = []string{ 73 + "application/json", 74 + "application/xml", 75 + "application/yaml", 76 + "application/x-yaml", 77 + "application/toml", 78 + "application/javascript", 79 + "application/ecmascript", 80 + } 81 + 82 + // isTextualMimeType returns true if the MIME type represents textual content 83 + // that should be served as text/plain for security reasons 84 + func isTextualMimeType(mimeType string) bool { 85 + return slices.Contains(textualMimeTypes, mimeType) 86 + }
+85
knotmirror/xrpc/git_getBranch.go
··· 1 + package xrpc 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "net/http" 7 + "net/url" 8 + "time" 9 + 10 + "github.com/bluesky-social/indigo/atproto/atclient" 11 + "github.com/bluesky-social/indigo/atproto/syntax" 12 + "tangled.org/core/api/tangled" 13 + "tangled.org/core/knotserver/git" 14 + ) 15 + 16 + // TODO: maybe rename to `sh.tangled.repo.temp.getCommit`? 17 + // then, we should ensure the given `ref` is valid 18 + func (x *Xrpc) GetBranch(w http.ResponseWriter, r *http.Request) { 19 + var ( 20 + repoQuery = r.URL.Query().Get("repo") 21 + nameQuery = r.URL.Query().Get("name") 22 + ) 23 + 24 + repo, err := syntax.ParseATURI(repoQuery) 25 + if err != nil || repo.RecordKey() == "" { 26 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) 27 + return 28 + } 29 + 30 + if nameQuery == "" { 31 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing name parameter"}) 32 + return 33 + } 34 + branchName, _ := url.PathUnescape(nameQuery) 35 + 36 + l := x.logger.With("repo", repo, "branch", branchName) 37 + 38 + out, err := x.getBranch(r.Context(), repo, branchName) 39 + if err != nil { 40 + // TODO: better error return 41 + l.Error("failed to get branch", "err", err) 42 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get branch"}) 43 + return 44 + } 45 + writeJson(w, http.StatusOK, out) 46 + } 47 + 48 + func (x *Xrpc) getBranch(ctx context.Context, repo syntax.ATURI, branchName string) (*tangled.GitTempGetBranch_Output, error) { 49 + repoPath, err := x.makeRepoPath(ctx, repo) 50 + if err != nil { 51 + return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err) 52 + } 53 + 54 + gr, err := git.PlainOpen(repoPath) 55 + if err != nil { 56 + return nil, fmt.Errorf("failed to open git repo: %w", err) 57 + } 58 + 59 + ref, err := gr.Branch(branchName) 60 + if err != nil { 61 + return nil, fmt.Errorf("getting branch '%s': %w", branchName, err) 62 + } 63 + 64 + commit, err := gr.Commit(ref.Hash()) 65 + if err != nil { 66 + return nil, fmt.Errorf("getting commit '%s': %w", ref.Hash(), err) 67 + } 68 + 69 + out := tangled.GitTempGetBranch_Output{ 70 + Name: ref.Name().Short(), 71 + Hash: ref.Hash().String(), 72 + When: commit.Author.When.Format(time.RFC3339), 73 + Author: &tangled.GitTempDefs_Signature{ 74 + Name: commit.Author.Name, 75 + Email: commit.Author.Email, 76 + When: commit.Author.When.Format(time.RFC3339), 77 + }, 78 + } 79 + 80 + if commit.Message != "" { 81 + out.Message = &commit.Message 82 + } 83 + 84 + return &out, nil 85 + }
+92
knotmirror/xrpc/git_getTag.go
··· 1 + package xrpc 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "net/http" 7 + 8 + "github.com/bluesky-social/indigo/atproto/atclient" 9 + "github.com/bluesky-social/indigo/atproto/syntax" 10 + "github.com/go-git/go-git/v5/plumbing" 11 + "github.com/go-git/go-git/v5/plumbing/object" 12 + "tangled.org/core/knotserver/git" 13 + "tangled.org/core/types" 14 + ) 15 + 16 + func (x *Xrpc) GetTag(w http.ResponseWriter, r *http.Request) { 17 + var ( 18 + repoQuery = r.URL.Query().Get("repo") 19 + tagName = r.URL.Query().Get("tag") 20 + ) 21 + 22 + repo, err := syntax.ParseATURI(repoQuery) 23 + if err != nil || repo.RecordKey() == "" { 24 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) 25 + return 26 + } 27 + 28 + if tagName == "" { 29 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing 'tag' parameter"}) 30 + return 31 + } 32 + 33 + l := x.logger.With("repo", repo, "tag", tagName) 34 + 35 + out, err := x.getTag(r.Context(), repo, tagName) 36 + if err != nil { 37 + // TODO: better error return 38 + l.Error("failed to get tag", "err", err) 39 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get tag"}) 40 + return 41 + } 42 + writeJson(w, http.StatusOK, out) 43 + } 44 + 45 + func (x *Xrpc) getTag(ctx context.Context, repo syntax.ATURI, tagName string) (*types.RepoTagResponse, error) { 46 + repoPath, err := x.makeRepoPath(ctx, repo) 47 + if err != nil { 48 + return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err) 49 + } 50 + 51 + gr, err := git.PlainOpen(repoPath) 52 + if err != nil { 53 + return nil, fmt.Errorf("failed to open git repo: %w", err) 54 + } 55 + 56 + // if this is not already formatted as refs/tags/v0.1.0, then format it 57 + if !plumbing.ReferenceName(tagName).IsTag() { 58 + tagName = plumbing.NewTagReferenceName(tagName).String() 59 + } 60 + 61 + tag, err := func() (object.Tag, error) { 62 + tags, err := gr.Tags(&git.TagsOptions{ 63 + Pattern: tagName, 64 + }) 65 + if err != nil { 66 + return object.Tag{}, err 67 + } 68 + if len(tags) != 1 { 69 + return object.Tag{}, fmt.Errorf("expected 1 tag to be returned, got %d tags", len(tags)) 70 + } 71 + return tags[0], nil 72 + }() 73 + if err != nil { 74 + return nil, fmt.Errorf("getting tag: %w", err) 75 + } 76 + 77 + var target *object.Tag 78 + if tag.Target != plumbing.ZeroHash { 79 + target = &tag 80 + } 81 + 82 + return &types.RepoTagResponse{ 83 + Tag: &types.TagReference{ 84 + Tag: target, 85 + Reference: types.Reference{ 86 + Name: tag.Name, 87 + Hash: tag.Hash.String(), 88 + }, 89 + Message: tag.Message, 90 + }, 91 + }, nil 92 + }
+118
knotmirror/xrpc/git_getTree.go
··· 1 + package xrpc 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "net/http" 7 + "path/filepath" 8 + "time" 9 + "unicode/utf8" 10 + 11 + "github.com/bluesky-social/indigo/atproto/atclient" 12 + "github.com/bluesky-social/indigo/atproto/syntax" 13 + "tangled.org/core/api/tangled" 14 + "tangled.org/core/appview/pages/markup" 15 + "tangled.org/core/knotserver/git" 16 + ) 17 + 18 + func (x *Xrpc) GetTree(w http.ResponseWriter, r *http.Request) { 19 + var ( 20 + repoQuery = r.URL.Query().Get("repo") 21 + ref = r.URL.Query().Get("ref") // ref can be empty (git.Open handles this) 22 + path = r.URL.Query().Get("path") // path can be empty (defaults to root) 23 + ) 24 + 25 + repo, err := syntax.ParseATURI(repoQuery) 26 + if err != nil || repo.RecordKey() == "" { 27 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) 28 + return 29 + } 30 + 31 + l := x.logger.With("repo", repo, "ref", ref, "path", path) 32 + 33 + out, err := x.getTree(r.Context(), repo, ref, path) 34 + if err != nil { 35 + // TODO: better error return 36 + l.Error("failed to get tree", "err", err) 37 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get tree"}) 38 + return 39 + } 40 + writeJson(w, http.StatusOK, out) 41 + } 42 + 43 + func (x *Xrpc) getTree(ctx context.Context, repo syntax.ATURI, ref, path string) (*tangled.GitTempGetTree_Output, error) { 44 + repoPath, err := x.makeRepoPath(ctx, repo) 45 + if err != nil { 46 + return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err) 47 + } 48 + 49 + gr, err := git.Open(repoPath, ref) 50 + if err != nil { 51 + return nil, fmt.Errorf("opening git repo: %w", err) 52 + } 53 + 54 + files, err := gr.FileTree(ctx, path) 55 + if err != nil { 56 + return nil, fmt.Errorf("reading file tree: %w", err) 57 + } 58 + 59 + // if any of these files are a readme candidate, pass along its blob contents too 60 + var readmeFileName string 61 + var readmeContents string 62 + for _, file := range files { 63 + if markup.IsReadmeFile(file.Name) { 64 + contents, err := gr.RawContent(filepath.Join(path, file.Name)) 65 + if err != nil { 66 + x.logger.Error("failed to read contents of file", "path", path, "file", file.Name) 67 + } 68 + 69 + if utf8.Valid(contents) { 70 + readmeFileName = file.Name 71 + readmeContents = string(contents) 72 + break 73 + } 74 + } 75 + } 76 + 77 + // convert NiceTree -> tangled.RepoTempGetTree_TreeEntry 78 + treeEntries := make([]*tangled.GitTempGetTree_TreeEntry, len(files)) 79 + for i, file := range files { 80 + entry := &tangled.GitTempGetTree_TreeEntry{ 81 + Name: file.Name, 82 + Mode: file.Mode, 83 + Size: file.Size, 84 + } 85 + if file.LastCommit != nil { 86 + entry.Last_commit = &tangled.GitTempGetTree_LastCommit{ 87 + Hash: file.LastCommit.Hash.String(), 88 + Message: file.LastCommit.Message, 89 + When: file.LastCommit.When.Format(time.RFC3339), 90 + } 91 + } 92 + treeEntries[i] = entry 93 + } 94 + 95 + var parentPtr *string 96 + if path != "" { 97 + parentPtr = &path 98 + } 99 + 100 + var dotdotPtr *string 101 + if path != "" { 102 + dotdot := filepath.Dir(path) 103 + if dotdot != "." { 104 + dotdotPtr = &dotdot 105 + } 106 + } 107 + 108 + return &tangled.GitTempGetTree_Output{ 109 + Ref: ref, 110 + Parent: parentPtr, 111 + Dotdot: dotdotPtr, 112 + Files: treeEntries, 113 + Readme: &tangled.GitTempGetTree_Readme{ 114 + Filename: readmeFileName, 115 + Contents: readmeContents, 116 + }, 117 + }, nil 118 + }
+95
knotmirror/xrpc/git_listBranches.go
··· 1 + package xrpc 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "net/http" 7 + "path/filepath" 8 + "strconv" 9 + 10 + "github.com/bluesky-social/indigo/atproto/atclient" 11 + "github.com/bluesky-social/indigo/atproto/syntax" 12 + "tangled.org/core/knotserver/git" 13 + "tangled.org/core/types" 14 + ) 15 + 16 + func (x *Xrpc) ListBranches(w http.ResponseWriter, r *http.Request) { 17 + var ( 18 + repoQuery = r.URL.Query().Get("repo") 19 + limitQuery = r.URL.Query().Get("limit") 20 + cursorQuery = r.URL.Query().Get("cursor") 21 + ) 22 + 23 + repo, err := syntax.ParseATURI(repoQuery) 24 + if err != nil || repo.RecordKey() == "" { 25 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) 26 + return 27 + } 28 + 29 + limit := 50 30 + if limitQuery != "" { 31 + limit, err = strconv.Atoi(limitQuery) 32 + if err != nil || limit < 1 || limit > 1000 { 33 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("limit parameter invalid: %s", limitQuery)}) 34 + return 35 + } 36 + } 37 + 38 + var cursor int64 39 + if cursorQuery != "" { 40 + cursor, err = strconv.ParseInt(cursorQuery, 10, 64) 41 + if err != nil || cursor < 0 { 42 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("cursor parameter invalid: %s", cursorQuery)}) 43 + return 44 + } 45 + } 46 + 47 + l := x.logger.With("repo", repoQuery, "limit", limit, "cursor", cursor) 48 + 49 + out, err := x.listBranches(r.Context(), repo, limit, cursor) 50 + if err != nil { 51 + // TODO: better error return 52 + l.Error("failed to list branches", "err", err) 53 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to list branches"}) 54 + return 55 + } 56 + writeJson(w, http.StatusOK, out) 57 + } 58 + 59 + func (x *Xrpc) listBranches(ctx context.Context, repo syntax.ATURI, limit int, cursor int64) (*types.RepoBranchesResponse, error) { 60 + repoPath, err := x.makeRepoPath(ctx, repo) 61 + if err != nil { 62 + return nil, fmt.Errorf("resolving repo at-uri: %w", err) 63 + } 64 + 65 + gr, err := git.PlainOpen(repoPath) 66 + if err != nil { 67 + return nil, fmt.Errorf("opening git repo: %w", err) 68 + } 69 + 70 + branches, err := gr.Branches(&git.BranchesOptions{ 71 + Limit: limit, 72 + Offset: int(cursor), 73 + }) 74 + if err != nil { 75 + return nil, fmt.Errorf("listing git branches: %w", err) 76 + } 77 + 78 + return &types.RepoBranchesResponse{ 79 + // TODO: include default branch and cursor 80 + Branches: branches, 81 + }, nil 82 + } 83 + 84 + func (x *Xrpc) makeRepoPath(ctx context.Context, repo syntax.ATURI) (string, error) { 85 + id, err := x.resolver.ResolveIdent(ctx, repo.Authority().String()) 86 + if err != nil { 87 + return "", err 88 + } 89 + 90 + return filepath.Join( 91 + x.cfg.GitRepoBasePath, 92 + id.DID.String(), 93 + repo.RecordKey().String(), 94 + ), nil 95 + }
+95
knotmirror/xrpc/git_listCommits.go
··· 1 + package xrpc 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "net/http" 7 + "strconv" 8 + 9 + "github.com/bluesky-social/indigo/atproto/atclient" 10 + "github.com/bluesky-social/indigo/atproto/syntax" 11 + "tangled.org/core/knotserver/git" 12 + "tangled.org/core/types" 13 + ) 14 + 15 + func (x *Xrpc) ListCommits(w http.ResponseWriter, r *http.Request) { 16 + var ( 17 + repoQuery = r.URL.Query().Get("repo") 18 + ref = r.URL.Query().Get("ref") // ref can be empty (git.Open handles this) 19 + limitQuery = r.URL.Query().Get("limit") 20 + cursorQuery = r.URL.Query().Get("cursor") 21 + ) 22 + 23 + repo, err := syntax.ParseATURI(repoQuery) 24 + if err != nil || repo.RecordKey() == "" { 25 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) 26 + return 27 + } 28 + 29 + limit := 50 30 + if limitQuery != "" { 31 + limit, err = strconv.Atoi(limitQuery) 32 + if err != nil || limit < 1 || limit > 1000 { 33 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("limit parameter invalid: %s", limitQuery)}) 34 + return 35 + } 36 + } 37 + 38 + var cursor int64 39 + if cursorQuery != "" { 40 + cursor, err = strconv.ParseInt(cursorQuery, 10, 64) 41 + if err != nil || cursor < 0 { 42 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("cursor parameter invalid: %s", cursorQuery)}) 43 + return 44 + } 45 + } 46 + 47 + l := x.logger.With("repo", repo, "ref", ref) 48 + 49 + out, err := x.listCommits(r.Context(), repo, ref, limit, cursor) 50 + if err != nil { 51 + // TODO: better error return 52 + l.Error("failed to list commits", "err", err) 53 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to list commits"}) 54 + return 55 + } 56 + writeJson(w, http.StatusOK, out) 57 + } 58 + 59 + func (x *Xrpc) listCommits(ctx context.Context, repo syntax.ATURI, ref string, limit int, cursor int64) (*types.RepoLogResponse, error) { 60 + repoPath, err := x.makeRepoPath(ctx, repo) 61 + if err != nil { 62 + return nil, fmt.Errorf("resolving repo at-uri: %w", err) 63 + } 64 + 65 + gr, err := git.Open(repoPath, ref) 66 + if err != nil { 67 + return nil, fmt.Errorf("opening git repo: %w", err) 68 + } 69 + 70 + offset := int(cursor) 71 + 72 + commits, err := gr.Commits(offset, limit) 73 + if err != nil { 74 + return nil, fmt.Errorf("listing git commits: %w", err) 75 + } 76 + 77 + tcommits := make([]types.Commit, len(commits)) 78 + for i, c := range commits { 79 + tcommits[i].FromGoGitCommit(c) 80 + } 81 + 82 + total, err := gr.TotalCommits() 83 + if err != nil { 84 + return nil, fmt.Errorf("counting total commits: %w", err) 85 + } 86 + 87 + return &types.RepoLogResponse{ 88 + Commits: tcommits, 89 + Ref: ref, 90 + Page: (offset / limit) + 1, 91 + PerPage: limit, 92 + Total: total, 93 + Log: true, 94 + }, nil 95 + }
+86
knotmirror/xrpc/git_listLanguages.go
··· 1 + package xrpc 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "math" 7 + "net/http" 8 + "time" 9 + 10 + "github.com/bluesky-social/indigo/atproto/atclient" 11 + "github.com/bluesky-social/indigo/atproto/syntax" 12 + "tangled.org/core/api/tangled" 13 + "tangled.org/core/knotserver/git" 14 + ) 15 + 16 + func (x *Xrpc) ListLanguages(w http.ResponseWriter, r *http.Request) { 17 + var ( 18 + repoQuery = r.URL.Query().Get("repo") 19 + ref = r.URL.Query().Get("ref") 20 + ) 21 + l := x.logger.With("repo", repoQuery, "ref", ref) 22 + 23 + repo, err := syntax.ParseATURI(repoQuery) 24 + if err != nil || repo.RecordKey() == "" { 25 + l.Error("invalid repo at-uri", "err", err) 26 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) 27 + return 28 + } 29 + 30 + out, err := x.listLanguages(r.Context(), repo, ref) 31 + if err != nil { 32 + l.Error("failed to list languages", "err", err) 33 + writeErr(w, err) 34 + return 35 + } 36 + 37 + writeJson(w, http.StatusOK, out) 38 + } 39 + 40 + func (x *Xrpc) listLanguages(ctx context.Context, repo syntax.ATURI, ref string) (*tangled.GitTempListLanguages_Output, error) { 41 + repoPath, err := x.makeRepoPath(ctx, repo) 42 + if err != nil { 43 + return nil, fmt.Errorf("resolving repo at-uri: %w", err) 44 + } 45 + 46 + gr, err := git.Open(repoPath, ref) 47 + if err != nil { 48 + return nil, &atclient.APIError{StatusCode: http.StatusNotFound, Name: "RepoNotFound", Message: "failed to find git repo"} 49 + } 50 + 51 + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) 52 + defer cancel() 53 + 54 + sizes, err := gr.AnalyzeLanguages(ctx) 55 + if err != nil { 56 + return nil, fmt.Errorf("analyzing languages: %w", err) 57 + } 58 + 59 + return &tangled.GitTempListLanguages_Output{ 60 + Ref: ref, 61 + Languages: sizesToLanguages(sizes), 62 + }, nil 63 + } 64 + 65 + func sizesToLanguages(sizes git.LangBreakdown) []*tangled.GitTempListLanguages_Language { 66 + var apiLanguages []*tangled.GitTempListLanguages_Language 67 + var totalSize int64 68 + for _, size := range sizes { 69 + totalSize += size 70 + } 71 + 72 + for name, size := range sizes { 73 + percentagef64 := float64(size) / float64(totalSize) * 100 74 + percentage := math.Round(percentagef64) 75 + 76 + lang := &tangled.GitTempListLanguages_Language{ 77 + Name: name, 78 + Size: size, 79 + Percentage: int64(percentage), 80 + } 81 + 82 + apiLanguages = append(apiLanguages, lang) 83 + } 84 + 85 + return apiLanguages 86 + }
+98
knotmirror/xrpc/git_listTags.go
··· 1 + package xrpc 2 + 3 + import ( 4 + "context" 5 + "fmt" 6 + "net/http" 7 + "strconv" 8 + 9 + "github.com/bluesky-social/indigo/atproto/atclient" 10 + "github.com/bluesky-social/indigo/atproto/syntax" 11 + "github.com/go-git/go-git/v5/plumbing" 12 + "github.com/go-git/go-git/v5/plumbing/object" 13 + "tangled.org/core/knotserver/git" 14 + "tangled.org/core/types" 15 + ) 16 + 17 + func (x *Xrpc) ListTags(w http.ResponseWriter, r *http.Request) { 18 + var ( 19 + repoQuery = r.URL.Query().Get("repo") 20 + limitQuery = r.URL.Query().Get("limit") 21 + cursorQuery = r.URL.Query().Get("cursor") 22 + ) 23 + 24 + repo, err := syntax.ParseATURI(repoQuery) 25 + if err != nil || repo.RecordKey() == "" { 26 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) 27 + return 28 + } 29 + 30 + limit := 50 31 + if limitQuery != "" { 32 + limit, err = strconv.Atoi(limitQuery) 33 + if err != nil || limit < 1 || limit > 1000 { 34 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("limit parameter invalid: %s", limitQuery)}) 35 + return 36 + } 37 + } 38 + 39 + var cursor int64 40 + if cursorQuery != "" { 41 + cursor, err = strconv.ParseInt(cursorQuery, 10, 64) 42 + if err != nil || cursor < 0 { 43 + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("cursor parameter invalid: %s", cursorQuery)}) 44 + return 45 + } 46 + } 47 + 48 + l := x.logger.With("repo", repo, "limit", limit, "cursor", cursor) 49 + 50 + out, err := x.listTags(r.Context(), repo, limit, cursor) 51 + if err != nil { 52 + // TODO: better error return 53 + l.Error("failed to list tags", "err", err) 54 + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to list tags"}) 55 + return 56 + } 57 + writeJson(w, http.StatusOK, out) 58 + } 59 + 60 + func (x *Xrpc) listTags(ctx context.Context, repo syntax.ATURI, limit int, cursor int64) (*types.RepoTagsResponse, error) { 61 + repoPath, err := x.makeRepoPath(ctx, repo) 62 + if err != nil { 63 + return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err) 64 + } 65 + 66 + gr, err := git.PlainOpen(repoPath) 67 + if err != nil { 68 + return nil, fmt.Errorf("failed to open git repo: %w", err) 69 + } 70 + 71 + tags, err := gr.Tags(&git.TagsOptions{ 72 + Limit: limit, 73 + Offset: int(cursor), 74 + }) 75 + if err != nil { 76 + return nil, fmt.Errorf("failed to get git tags: %w", err) 77 + } 78 + 79 + rtags := make([]*types.TagReference, len(tags)) 80 + for i, tag := range tags { 81 + var target *object.Tag 82 + if tag.Target != plumbing.ZeroHash { 83 + target = &tag 84 + } 85 + rtags[i] = &types.TagReference{ 86 + Reference: types.Reference{ 87 + Name: tag.Name, 88 + Hash: tag.Hash.String(), 89 + }, 90 + Tag: target, 91 + Message: tag.Message, 92 + } 93 + } 94 + 95 + return &types.RepoTagsResponse{ 96 + Tags: rtags, 97 + }, nil 98 + }
+69
knotmirror/xrpc/xrpc.go
··· 1 + package xrpc 2 + 3 + import ( 4 + "database/sql" 5 + "encoding/json" 6 + "errors" 7 + "log/slog" 8 + "net/http" 9 + 10 + "github.com/bluesky-social/indigo/atproto/atclient" 11 + "github.com/go-chi/chi/v5" 12 + "tangled.org/core/api/tangled" 13 + "tangled.org/core/idresolver" 14 + "tangled.org/core/knotmirror/config" 15 + "tangled.org/core/log" 16 + ) 17 + 18 + type Xrpc struct { 19 + cfg *config.Config 20 + db *sql.DB 21 + resolver *idresolver.Resolver 22 + logger *slog.Logger 23 + } 24 + 25 + func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, resolver *idresolver.Resolver) *Xrpc { 26 + return &Xrpc{ 27 + cfg, 28 + db, 29 + resolver, 30 + log.SubLogger(logger, "xrpc"), 31 + } 32 + } 33 + 34 + func (x *Xrpc) Router() http.Handler { 35 + r := chi.NewRouter() 36 + 37 + r.Get("/"+tangled.GitTempGetArchiveNSID, x.GetArchive) 38 + r.Get("/"+tangled.GitTempGetBlobNSID, x.GetBlob) 39 + r.Get("/"+tangled.GitTempGetBranchNSID, x.GetBranch) 40 + // r.Get("/"+tangled.GitTempGetCommitNSID, x.GetCommit) // todo 41 + // r.Get("/"+tangled.GitTempGetDiffNSID, x.GetDiff) // todo 42 + // r.Get("/"+tangled.GitTempGetEntityNSID, x.GetEntity) // todo 43 + // r.Get("/"+tangled.GitTempGetHeadNSID, x.GetHead) // todo 44 + r.Get("/"+tangled.GitTempGetTagNSID, x.GetTag) // using types.Response 45 + r.Get("/"+tangled.GitTempGetTreeNSID, x.GetTree) 46 + r.Get("/"+tangled.GitTempListBranchesNSID, x.ListBranches) // wip, unknown output 47 + r.Get("/"+tangled.GitTempListCommitsNSID, x.ListCommits) 48 + r.Get("/"+tangled.GitTempListLanguagesNSID, x.ListLanguages) 49 + r.Get("/"+tangled.GitTempListTagsNSID, x.ListTags) 50 + 51 + return r 52 + } 53 + 54 + func writeJson(w http.ResponseWriter, status int, response any) error { 55 + w.Header().Set("Content-Type", "application/json") 56 + w.WriteHeader(status) 57 + if err := json.NewEncoder(w).Encode(response); err != nil { 58 + return err 59 + } 60 + return nil 61 + } 62 + 63 + func writeErr(w http.ResponseWriter, err error) error { 64 + var apiErr *atclient.APIError 65 + if errors.As(err, &apiErr) { 66 + return writeJson(w, apiErr.StatusCode, atclient.ErrorBody{Name: apiErr.Name, Message: apiErr.Message}) 67 + } 68 + return writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "internal server error"}) 69 + }
+14
knotserver/git/git.go
··· 199 199 return io.ReadAll(reader) 200 200 } 201 201 202 + func (g *GitRepo) File(path string) (*object.File, error) { 203 + c, err := g.r.CommitObject(g.h) 204 + if err != nil { 205 + return nil, fmt.Errorf("commit object: %w", err) 206 + } 207 + 208 + tree, err := c.Tree() 209 + if err != nil { 210 + return nil, fmt.Errorf("file tree: %w", err) 211 + } 212 + 213 + return tree.File(path) 214 + } 215 + 202 216 // read and parse .gitmodules 203 217 func (g *GitRepo) Submodules() (*config.Modules, error) { 204 218 c, err := g.r.CommitObject(g.h)