appview,knotserver: improve txn handling around repo creation and forking

+8 -2

appview/pages/templates/repo/fork.html

··· 5 5 <p class="text-xl font-bold dark:text-white">Fork {{ .RepoInfo.FullName }}</p> 6 6 </div> 7 7 <div class="p-6 bg-white dark:bg-gray-800 drop-shadow-sm rounded"> 8 - <form hx-post="/{{ .RepoInfo.FullName }}/fork" class="space-y-12" hx-swap="none"> 8 + <form hx-post="/{{ .RepoInfo.FullName }}/fork" class="space-y-12" hx-swap="none" hx-indicator="#spinner"> 9 9 <fieldset class="space-y-3"> 10 10 <legend class="dark:text-white">Select a knot to fork into</legend> 11 11 <div class="space-y-2"> ··· 30 30 </fieldset> 31 31 32 32 <div class="space-y-2"> 33 - <button type="submit" class="btn">fork repo</button> 33 + <button type="submit" class="btn-create flex items-center gap-2"> 34 + {{ i "git-fork" "w-4 h-4" }} 35 + fork repo 36 + <span id="spinner" class="group"> 37 + {{ i "loader-circle" "w-4 h-4 animate-spin hidden group-[.htmx-request]:inline" }} 38 + </span> 39 + </button> 34 40 <div id="repo" class="error"></div> 35 41 </div> 36 42 </form>

+1 -1

appview/pages/templates/repo/new.html

··· 63 63 <button type="submit" class="btn-create flex items-center gap-2"> 64 64 {{ i "book-plus" "w-4 h-4" }} 65 65 create repo 66 - <span id="create-pull-spinner" class="group"> 66 + <span id="spinner" class="group"> 67 67 {{ i "loader-circle" "w-4 h-4 animate-spin hidden group-[.htmx-request]:inline" }} 68 68 </span> 69 69 </button>

+102 -64

appview/repo/repo.go

··· 28 28 "tangled.sh/tangled.sh/core/appview/pages" 29 29 "tangled.sh/tangled.sh/core/appview/pages/markup" 30 30 "tangled.sh/tangled.sh/core/appview/reporesolver" 31 + xrpcclient "tangled.sh/tangled.sh/core/appview/xrpcclient" 31 32 "tangled.sh/tangled.sh/core/eventconsumer" 32 33 "tangled.sh/tangled.sh/core/idresolver" 33 34 "tangled.sh/tangled.sh/core/knotclient" ··· 1454 1453 }) 1455 1454 1456 1455 case http.MethodPost: 1456 + l := rp.logger.With("handler", "ForkRepo") 1457 1457 1458 - knot := r.FormValue("knot") 1459 - if knot == "" { 1458 + targetKnot := r.FormValue("knot") 1459 + if targetKnot == "" { 1460 1460 rp.pages.Notice(w, "repo", "Invalid form submission—missing knot domain.") 1461 1461 return 1462 1462 } 1463 + l = l.With("targetKnot", targetKnot) 1463 1464 1464 - ok, err := rp.enforcer.E.Enforce(user.Did, knot, knot, "repo:create") 1465 + ok, err := rp.enforcer.E.Enforce(user.Did, targetKnot, targetKnot, "repo:create") 1465 1466 if err != nil || !ok { 1466 1467 rp.pages.Notice(w, "repo", "You do not have permission to create a repo in this knot.") 1467 1468 return 1468 1469 } 1469 1470 1470 - forkName := fmt.Sprintf("%s", f.Name) 1471 - 1471 + // choose a name for a fork 1472 + forkName := f.Name 1472 1473 // this check is *only* to see if the forked repo name already exists 1473 1474 // in the user's account. 1474 1475 existingRepo, err := db.GetRepo(rp.db, user.Did, f.Name) ··· 1486 1483 // repo with this name already exists, append random string 1487 1484 forkName = fmt.Sprintf("%s-%s", forkName, randomString(3)) 1488 1485 } 1489 - client, err := rp.oauth.ServiceClient( 1490 - r, 1491 - oauth.WithService(knot), 1492 - oauth.WithLxm(tangled.RepoForkNSID), 1493 - oauth.WithDev(rp.config.Core.Dev), 1494 - ) 1486 + l = l.With("forkName", forkName) 1495 1487 1496 - if err != nil { 1497 - log.Printf("error creating client for knot server: %v", err) 1498 - rp.pages.Notice(w, "repo", "Failed to connect to knot server.") 1499 - return 1500 - } 1501 - 1502 - var uri string 1488 + uri := "https" 1503 1489 if rp.config.Core.Dev { 1504 1490 uri = "http" 1505 - } else { 1506 - uri = "https" 1507 1491 } 1492 + 1508 1493 forkSourceUrl := fmt.Sprintf("%s://%s/%s/%s", uri, f.Knot, f.OwnerDid(), f.Repo.Name) 1494 + l = l.With("cloneUrl", forkSourceUrl) 1495 + 1509 1496 sourceAt := f.RepoAt().String() 1510 1497 1498 + // create an atproto record for this fork 1511 1499 rkey := tid.TID() 1512 1500 repo := &db.Repo{ 1513 1501 Did: user.Did, 1514 1502 Name: forkName, 1515 - Knot: knot, 1503 + Knot: targetKnot, 1516 1504 Rkey: rkey, 1517 1505 Source: sourceAt, 1518 1506 } 1519 1507 1520 - tx, err := rp.db.BeginTx(r.Context(), nil) 1521 - if err != nil { 1522 - log.Println(err) 1523 - rp.pages.Notice(w, "repo", "Failed to save repository information.") 1524 - return 1525 - } 1526 - defer func() { 1527 - tx.Rollback() 1528 - err = rp.enforcer.E.LoadPolicy() 1529 - if err != nil { 1530 - log.Println("failed to rollback policies") 1531 - } 1532 - }() 1533 - 1534 - err = tangled.RepoFork( 1535 - r.Context(), 1536 - client, 1537 - &tangled.RepoFork_Input{ 1538 - Did: user.Did, 1539 - Name: &forkName, 1540 - Source: forkSourceUrl, 1541 - }, 1542 - ) 1543 - 1544 - if err != nil { 1545 - xe, err := xrpcerr.Unmarshal(err.Error()) 1546 - if err != nil { 1547 - log.Println(err) 1548 - rp.pages.Notice(w, "repo", "Failed to create repository on knot server.") 1549 - return 1550 - } 1551 - 1552 - log.Println(xe.Error()) 1553 - rp.pages.Notice(w, "repo", fmt.Sprintf("Failed to create repository on knot server: %s.", xe.Message)) 1554 - return 1555 - } 1556 - 1557 1508 xrpcClient, err := rp.oauth.AuthorizedClient(r) 1558 1509 if err != nil { 1559 - log.Println("failed to get authorized client", err) 1560 - rp.pages.Notice(w, "repo", "Failed to create repository.") 1510 + l.Error("failed to create xrpcclient", "err", err) 1511 + rp.pages.Notice(w, "repo", "Failed to fork repository.") 1561 1512 return 1562 1513 } 1563 1514 ··· 1530 1573 }}, 1531 1574 }) 1532 1575 if err != nil { 1533 - log.Printf("failed to create record: %s", err) 1576 + l.Error("failed to write to PDS", "err", err) 1534 1577 rp.pages.Notice(w, "repo", "Failed to announce repository creation.") 1535 1578 return 1536 1579 } 1537 - log.Println("created repo record: ", atresp.Uri) 1580 + 1581 + aturi := atresp.Uri 1582 + l = l.With("aturi", aturi) 1583 + l.Info("wrote to PDS") 1584 + 1585 + tx, err := rp.db.BeginTx(r.Context(), nil) 1586 + if err != nil { 1587 + l.Info("txn failed", "err", err) 1588 + rp.pages.Notice(w, "repo", "Failed to save repository information.") 1589 + return 1590 + } 1591 + 1592 + // The rollback function reverts a few things on failure: 1593 + // - the pending txn 1594 + // - the ACLs 1595 + // - the atproto record created 1596 + rollback := func() { 1597 + err1 := tx.Rollback() 1598 + err2 := rp.enforcer.E.LoadPolicy() 1599 + err3 := rollbackRecord(context.Background(), aturi, xrpcClient) 1600 + 1601 + // ignore txn complete errors, this is okay 1602 + if errors.Is(err1, sql.ErrTxDone) { 1603 + err1 = nil 1604 + } 1605 + 1606 + if errs := errors.Join(err1, err2, err3); errs != nil { 1607 + l.Error("failed to rollback changes", "errs", errs) 1608 + return 1609 + } 1610 + } 1611 + defer rollback() 1612 + 1613 + client, err := rp.oauth.ServiceClient( 1614 + r, 1615 + oauth.WithService(targetKnot), 1616 + oauth.WithLxm(tangled.RepoCreateNSID), 1617 + oauth.WithDev(rp.config.Core.Dev), 1618 + ) 1619 + if err != nil { 1620 + l.Error("could not create service client", "err", err) 1621 + rp.pages.Notice(w, "repo", "Failed to connect to knot server.") 1622 + return 1623 + } 1624 + 1625 + err = tangled.RepoCreate( 1626 + r.Context(), 1627 + client, 1628 + &tangled.RepoCreate_Input{ 1629 + Rkey: rkey, 1630 + Source: &forkSourceUrl, 1631 + }, 1632 + ) 1633 + if err := xrpcclient.HandleXrpcErr(err); err != nil { 1634 + rp.pages.Notice(w, "repo", err.Error()) 1635 + return 1636 + } 1538 1637 1539 1638 err = db.AddRepo(tx, repo) 1540 1639 if err != nil { ··· 1601 1588 1602 1589 // acls 1603 1590 p, _ := securejoin.SecureJoin(user.Did, forkName) 1604 - err = rp.enforcer.AddRepo(user.Did, knot, p) 1591 + err = rp.enforcer.AddRepo(user.Did, targetKnot, p) 1605 1592 if err != nil { 1606 1593 log.Println(err) 1607 1594 rp.pages.Notice(w, "repo", "Failed to set up repository permissions.") ··· 1622 1609 return 1623 1610 } 1624 1611 1612 + // reset the ATURI because the transaction completed successfully 1613 + aturi = "" 1614 + 1615 + rp.notifier.NewRepo(r.Context(), repo) 1625 1616 rp.pages.HxLocation(w, fmt.Sprintf("/@%s/%s", user.Handle, forkName)) 1626 - return 1627 1617 } 1618 + } 1619 + 1620 + // this is used to rollback changes made to the PDS 1621 + // 1622 + // it is a no-op if the provided ATURI is empty 1623 + func rollbackRecord(ctx context.Context, aturi string, xrpcc *xrpcclient.Client) error { 1624 + if aturi == "" { 1625 + return nil 1626 + } 1627 + 1628 + parsed := syntax.ATURI(aturi) 1629 + 1630 + collection := parsed.Collection().String() 1631 + repo := parsed.Authority().String() 1632 + rkey := parsed.RecordKey().String() 1633 + 1634 + _, err := xrpcc.RepoDeleteRecord(ctx, &comatproto.RepoDeleteRecord_Input{ 1635 + Collection: collection, 1636 + Repo: repo, 1637 + Rkey: rkey, 1638 + }) 1639 + return err 1628 1640 } 1629 1641 1630 1642 func (rp *Repo) RepoCompareNew(w http.ResponseWriter, r *http.Request) {

+89 -38

appview/state/state.go

··· 2 2 3 3 import ( 4 4 "context" 5 + "database/sql" 6 + "errors" 5 7 "fmt" 6 8 "log" 7 9 "log/slog" ··· 12 10 "time" 13 11 14 12 comatproto "github.com/bluesky-social/indigo/api/atproto" 13 + "github.com/bluesky-social/indigo/atproto/syntax" 15 14 lexutil "github.com/bluesky-social/indigo/lex/util" 16 15 securejoin "github.com/cyphar/filepath-securejoin" 17 16 "github.com/go-chi/chi/v5" ··· 28 25 "tangled.sh/tangled.sh/core/appview/pages" 29 26 posthogService "tangled.sh/tangled.sh/core/appview/posthog" 30 27 "tangled.sh/tangled.sh/core/appview/reporesolver" 28 + xrpcclient "tangled.sh/tangled.sh/core/appview/xrpcclient" 31 29 "tangled.sh/tangled.sh/core/eventconsumer" 32 30 "tangled.sh/tangled.sh/core/idresolver" 33 31 "tangled.sh/tangled.sh/core/jetstream" ··· 52 48 repoResolver *reporesolver.RepoResolver 53 49 knotstream *eventconsumer.Consumer 54 50 spindlestream *eventconsumer.Consumer 51 + logger *slog.Logger 55 52 } 56 53 57 54 func Make(ctx context.Context, config *config.Config) (*State, error) { ··· 157 152 repoResolver, 158 153 knotstream, 159 154 spindlestream, 155 + slog.Default(), 160 156 } 161 157 162 158 return state, nil ··· 297 291 }) 298 292 299 293 case http.MethodPost: 300 - user := s.oauth.GetUser(r) 294 + l := s.logger.With("handler", "NewRepo") 301 295 296 + user := s.oauth.GetUser(r) 297 + l = l.With("did", user.Did) 298 + l = l.With("handle", user.Handle) 299 + 300 + // form validation 302 301 domain := r.FormValue("domain") 303 302 if domain == "" { 304 303 s.pages.Notice(w, "repo", "Invalid form submission—missing knot domain.") 305 304 return 306 305 } 306 + l = l.With("knot", domain) 307 307 308 308 repoName := r.FormValue("name") 309 309 if repoName == "" { ··· 321 309 s.pages.Notice(w, "repo", err.Error()) 322 310 return 323 311 } 324 - 325 312 repoName = stripGitExt(repoName) 313 + l = l.With("repoName", repoName) 326 314 327 315 defaultBranch := r.FormValue("branch") 328 316 if defaultBranch == "" { 329 317 defaultBranch = "main" 330 318 } 319 + l = l.With("defaultBranch", defaultBranch) 331 320 332 321 description := r.FormValue("description") 333 322 323 + // ACL validation 334 324 ok, err := s.enforcer.E.Enforce(user.Did, domain, domain, "repo:create") 335 325 if err != nil || !ok { 326 + l.Info("unauthorized") 336 327 s.pages.Notice(w, "repo", "You do not have permission to create a repo in this knot.") 337 328 return 338 329 } 339 330 331 + // Check for existing repos 340 332 existingRepo, err := db.GetRepo(s.db, user.Did, repoName) 341 333 if err == nil && existingRepo != nil { 342 334 l.Info("repo exists") ··· 348 332 return 349 333 } 350 334 351 - client, err := s.oauth.ServiceClient( 352 - r, 353 - oauth.WithService(domain), 354 - oauth.WithLxm(tangled.RepoCreateNSID), 355 - oauth.WithDev(s.config.Core.Dev), 356 - ) 357 - 358 - if err != nil { 359 - s.pages.Notice(w, "repo", "Failed to connect to knot server.") 360 - return 361 - } 362 - 335 + // create atproto record for this repo 363 336 rkey := tid.TID() 364 337 repo := &db.Repo{ 365 338 Did: user.Did, ··· 360 355 361 356 xrpcClient, err := s.oauth.AuthorizedClient(r) 362 357 if err != nil { 358 + l.Info("PDS write failed", "err", err) 363 359 s.pages.Notice(w, "repo", "Failed to write record to PDS.") 364 360 return 365 361 } ··· 379 373 }}, 380 374 }) 381 375 if err != nil { 382 - log.Printf("failed to create record: %s", err) 376 + l.Info("PDS write failed", "err", err) 383 377 s.pages.Notice(w, "repo", "Failed to announce repository creation.") 384 378 return 385 379 } 386 - log.Println("created repo record: ", atresp.Uri) 380 + 381 + aturi := atresp.Uri 382 + l = l.With("aturi", aturi) 383 + l.Info("wrote to PDS") 387 384 388 385 tx, err := s.db.BeginTx(r.Context(), nil) 389 386 if err != nil { 390 - log.Println(err) 387 + l.Info("txn failed", "err", err) 391 388 s.pages.Notice(w, "repo", "Failed to save repository information.") 392 389 return 393 390 } 394 - defer func() { 395 - tx.Rollback() 396 - err = s.enforcer.E.LoadPolicy() 397 - if err != nil { 398 - log.Println("failed to rollback policies") 391 + 392 + // The rollback function reverts a few things on failure: 393 + // - the pending txn 394 + // - the ACLs 395 + // - the atproto record created 396 + rollback := func() { 397 + err1 := tx.Rollback() 398 + err2 := s.enforcer.E.LoadPolicy() 399 + err3 := rollbackRecord(context.Background(), aturi, xrpcClient) 400 + 401 + // ignore txn complete errors, this is okay 402 + if errors.Is(err1, sql.ErrTxDone) { 403 + err1 = nil 399 404 } 400 - }() 405 + 406 + if errs := errors.Join(err1, err2, err3); errs != nil { 407 + l.Error("failed to rollback changes", "errs", errs) 408 + return 409 + } 410 + } 411 + defer rollback() 412 + 413 + client, err := s.oauth.ServiceClient( 414 + r, 415 + oauth.WithService(domain), 416 + oauth.WithLxm(tangled.RepoCreateNSID), 417 + oauth.WithDev(s.config.Core.Dev), 418 + ) 419 + if err != nil { 420 + l.Error("service auth failed", "err", err) 421 + s.pages.Notice(w, "repo", "Failed to reach PDS.") 422 + return 423 + } 401 424 402 425 xe := tangled.RepoCreate( 403 426 r.Context(), ··· 436 401 }, 437 402 ) 438 403 if err != nil { 439 - xe, err := xrpcerr.Unmarshal(err.Error()) 440 - if err != nil { 441 - log.Println(err) 442 - s.pages.Notice(w, "repo", "Failed to create repository on knot server.") 443 - return 444 - } 445 - 446 - log.Println(xe.Error()) 447 - s.pages.Notice(w, "repo", fmt.Sprintf("Failed to create repository on knot server: %s.", xe.Message)) 404 + l.Error("xrpc request failed", "err", err) 405 + s.pages.Notice(w, "repo", fmt.Sprintf("Failed to create repository on knot server: %s.", err.Error())) 448 406 return 449 407 } 450 408 451 409 err = db.AddRepo(tx, repo) 452 410 if err != nil { 453 - log.Println(err) 411 + l.Error("db write failed", "err", err) 454 412 s.pages.Notice(w, "repo", "Failed to save repository information.") 455 413 return 456 414 } ··· 452 424 p, _ := securejoin.SecureJoin(user.Did, repoName) 453 425 err = s.enforcer.AddRepo(user.Did, domain, p) 454 426 if err != nil { 455 - log.Println(err) 427 + l.Error("acl setup failed", "err", err) 456 428 s.pages.Notice(w, "repo", "Failed to set up repository permissions.") 457 429 return 458 430 } 459 431 460 432 err = tx.Commit() 461 433 if err != nil { 462 - log.Println("failed to commit changes", err) 434 + l.Error("txn commit failed", "err", err) 463 435 http.Error(w, err.Error(), http.StatusInternalServerError) 464 436 return 465 437 } 466 438 467 439 err = s.enforcer.E.SavePolicy() 468 440 if err != nil { 469 - log.Println("failed to update ACLs", err) 441 + l.Error("acl save failed", "err", err) 470 442 http.Error(w, err.Error(), http.StatusInternalServerError) 471 443 return 472 444 } 473 445 474 - s.notifier.NewRepo(r.Context(), repo) 446 + // reset the ATURI because the transaction completed successfully 447 + aturi = "" 475 448 449 + s.notifier.NewRepo(r.Context(), repo) 476 450 s.pages.HxLocation(w, fmt.Sprintf("/@%s/%s", user.Handle, repoName)) 477 - return 478 451 } 452 + } 453 + 454 + // this is used to rollback changes made to the PDS 455 + // 456 + // it is a no-op if the provided ATURI is empty 457 + func rollbackRecord(ctx context.Context, aturi string, xrpcc *xrpcclient.Client) error { 458 + if aturi == "" { 459 + return nil 460 + } 461 + 462 + parsed := syntax.ATURI(aturi) 463 + 464 + collection := parsed.Collection().String() 465 + repo := parsed.Authority().String() 466 + rkey := parsed.RecordKey().String() 467 + 468 + _, err := xrpcc.RepoDeleteRecord(ctx, &comatproto.RepoDeleteRecord_Input{ 469 + Collection: collection, 470 + Repo: repo, 471 + Rkey: rkey, 472 + }) 473 + return err 479 474 }

+8 -5

knotserver/xrpc/router.go knotserver/xrpc/xrpc.go

··· 31 31 32 32 func (x *Xrpc) Router() http.Handler { 33 33 r := chi.NewRouter() 34 + 34 35 r.Group(func(r chi.Router) { 35 36 r.Use(x.ServiceAuth.VerifyServiceAuth) 36 37 37 38 r.Post("/"+tangled.RepoSetDefaultBranchNSID, x.SetDefaultBranch) 38 39 r.Post("/"+tangled.RepoCreateNSID, x.CreateRepo) 39 - r.Post("/"+tangled.RepoDeleteNSID, x.DeleteRepo) 40 - r.Post("/"+tangled.RepoForkNSID, x.ForkRepo) 41 40 r.Post("/"+tangled.RepoForkStatusNSID, x.ForkStatus) 42 41 r.Post("/"+tangled.RepoForkSyncNSID, x.ForkSync) 43 - 44 42 r.Post("/"+tangled.RepoHiddenRefNSID, x.HiddenRef) 45 - 46 43 r.Post("/"+tangled.RepoMergeNSID, x.Merge) 47 - r.Post("/"+tangled.RepoMergeCheckNSID, x.MergeCheck) 48 44 }) 45 + 46 + // merge check is an open endpoint 47 + // 48 + // TODO: should we constrain this more? 49 + // - we can calculate on PR submit/resubmit/gitRefUpdate etc. 50 + // - use ETags on clients to keep requests to a minimum 51 + r.Post("/"+tangled.RepoMergeCheckNSID, x.MergeCheck) 49 52 return r 50 53 } 51 54

Configure Feed

Configure Feed