forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
appview: use repo_language table to access language info
this falls back to calling RepoLanguages on the knot, and caches that
info at the appview.
Signed-off-by: oppiliappan <me@oppi.li>
authored by
oppiliappan
and committed by
Tangled
812a7689
2001ea37
+100
-37
+44
-25
appview/repo/index.go
+44
-25
appview/repo/index.go
···
123
123
}
124
124
}
125
125
126
-
languageInfo, err := getLanguageInfo(f, signedClient, ref)
126
+
languageInfo, err := rp.getLanguageInfo(f, signedClient, ref)
127
127
if err != nil {
128
128
log.Printf("failed to compute language percentages: %s", err)
129
129
// non-fatal
···
153
153
Languages: languageInfo,
154
154
Pipelines: pipelines,
155
155
})
156
-
return
157
156
}
158
157
159
-
func getLanguageInfo(
158
+
func (rp *Repo) getLanguageInfo(
160
159
f *reporesolver.ResolvedRepo,
161
160
signedClient *knotclient.SignedClient,
162
161
ref string,
163
162
) ([]types.RepoLanguageDetails, error) {
164
-
repoLanguages, err := signedClient.RepoLanguages(f.OwnerDid(), f.RepoName, ref)
165
-
if err != nil {
166
-
return []types.RepoLanguageDetails{}, err
167
-
}
168
-
if repoLanguages == nil {
169
-
repoLanguages = &types.RepoLanguageResponse{Languages: make(map[string]int64)}
163
+
// first attempt to fetch from db
164
+
langs, err := db.GetRepoLanguages(
165
+
rp.db,
166
+
db.FilterEq("repo_at", f.RepoAt),
167
+
db.FilterEq("ref", ref),
168
+
)
169
+
170
+
if err != nil || langs == nil {
171
+
// non-fatal, fetch langs from ks
172
+
ls, err := signedClient.RepoLanguages(f.OwnerDid(), f.RepoName, ref)
173
+
if err != nil {
174
+
return nil, err
175
+
}
176
+
if ls == nil {
177
+
return nil, nil
178
+
}
179
+
for l, s := range ls.Languages {
180
+
langs = append(langs, db.RepoLanguage{
181
+
RepoAt: f.RepoAt,
182
+
Ref: ref,
183
+
Language: l,
184
+
Bytes: s,
185
+
})
186
+
}
187
+
188
+
// update appview's cache
189
+
err = db.InsertRepoLanguages(rp.db, langs)
190
+
if err != nil {
191
+
// non-fatal
192
+
log.Println("failed to cache lang results", err)
193
+
}
170
194
}
171
195
172
-
var totalSize int64
173
-
for _, fileSize := range repoLanguages.Languages {
174
-
totalSize += fileSize
196
+
var total int64
197
+
for _, l := range langs {
198
+
total += l.Bytes
175
199
}
176
200
177
201
var languageStats []types.RepoLanguageDetails
178
-
var otherPercentage float32 = 0
179
-
180
-
for lang, size := range repoLanguages.Languages {
181
-
percentage := (float32(size) / float32(totalSize)) * 100
182
-
183
-
if percentage <= 0.5 {
184
-
otherPercentage += percentage
185
-
continue
186
-
}
187
-
188
-
color := enry.GetColor(lang)
189
-
190
-
languageStats = append(languageStats, types.RepoLanguageDetails{Name: lang, Percentage: percentage, Color: color})
202
+
for _, l := range langs {
203
+
percentage := float32(l.Bytes) / float32(total) * 100
204
+
color := enry.GetColor(l.Language)
205
+
languageStats = append(languageStats, types.RepoLanguageDetails{
206
+
Name: l.Language,
207
+
Percentage: percentage,
208
+
Color: color,
209
+
})
191
210
}
192
211
193
212
sort.Slice(languageStats, func(i, j int) bool {
+56
-12
appview/state/knotstream.go
+56
-12
appview/state/knotstream.go
···
3
3
import (
4
4
"context"
5
5
"encoding/json"
6
+
"errors"
6
7
"fmt"
7
8
"slices"
8
9
"time"
···
19
18
"tangled.sh/tangled.sh/core/workflow"
20
19
21
20
"github.com/bluesky-social/indigo/atproto/syntax"
21
+
"github.com/go-git/go-git/v5/plumbing"
22
22
"github.com/posthog/posthog-go"
23
23
)
24
24
···
41
39
42
40
cfg := ec.ConsumerConfig{
43
41
Sources: srcs,
44
-
ProcessFunc: knotIngester(ctx, d, enforcer, posthog, c.Core.Dev),
42
+
ProcessFunc: knotIngester(d, enforcer, posthog, c.Core.Dev),
45
43
RetryInterval: c.Knotstream.RetryInterval,
46
44
MaxRetryInterval: c.Knotstream.MaxRetryInterval,
47
45
ConnectionTimeout: c.Knotstream.ConnectionTimeout,
···
55
53
return ec.NewConsumer(cfg), nil
56
54
}
57
55
58
-
func knotIngester(ctx context.Context, d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client, dev bool) ec.ProcessFunc {
56
+
func knotIngester(d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client, dev bool) ec.ProcessFunc {
59
57
return func(ctx context.Context, source ec.Source, msg ec.Message) error {
60
58
switch msg.Nsid {
61
59
case tangled.GitRefUpdateNSID:
···
83
81
return fmt.Errorf("%s does not belong to %s, something is fishy", record.CommitterDid, source.Key())
84
82
}
85
83
84
+
err1 := populatePunchcard(d, record)
85
+
err2 := updateRepoLanguages(d, record)
86
+
87
+
var err3 error
88
+
if !dev {
89
+
err3 = pc.Enqueue(posthog.Capture{
90
+
DistinctId: record.CommitterDid,
91
+
Event: "git_ref_update",
92
+
})
93
+
}
94
+
95
+
return errors.Join(err1, err2, err3)
96
+
}
97
+
98
+
func populatePunchcard(d *db.DB, record tangled.GitRefUpdate) error {
86
99
knownEmails, err := db.GetAllEmails(d, record.CommitterDid)
87
100
if err != nil {
88
101
return err
89
102
}
103
+
90
104
count := 0
91
105
for _, ke := range knownEmails {
92
106
if record.Meta == nil {
···
126
108
Date: time.Now(),
127
109
Count: count,
128
110
}
129
-
if err := db.AddPunch(d, punch); err != nil {
130
-
return err
111
+
return db.AddPunch(d, punch)
112
+
}
113
+
114
+
func updateRepoLanguages(d *db.DB, record tangled.GitRefUpdate) error {
115
+
if record.Meta == nil && record.Meta.LangBreakdown == nil {
116
+
return fmt.Errorf("empty language data for repo: %s/%s", record.RepoDid, record.RepoName)
131
117
}
132
118
133
-
if !dev {
134
-
err = pc.Enqueue(posthog.Capture{
135
-
DistinctId: record.CommitterDid,
136
-
Event: "git_ref_update",
137
-
})
138
-
if err != nil {
139
-
// non-fatal, TODO: log this
119
+
repos, err := db.GetRepos(
120
+
d,
121
+
db.FilterEq("did", record.RepoDid),
122
+
db.FilterEq("name", record.RepoName),
123
+
)
124
+
if err != nil {
125
+
return fmt.Errorf("failed to look for repo in DB (%s/%s): %w", record.RepoDid, record.RepoName, err)
126
+
}
127
+
if len(repos) != 1 {
128
+
return fmt.Errorf("incorrect number of repos returned: %d (expected 1)", len(repos))
129
+
}
130
+
repo := repos[0]
131
+
132
+
ref := plumbing.ReferenceName(record.Ref)
133
+
if !ref.IsBranch() {
134
+
return fmt.Errorf("%s is not a valid reference name", ref)
135
+
}
136
+
137
+
var langs []db.RepoLanguage
138
+
for _, l := range record.Meta.LangBreakdown.Inputs {
139
+
if l == nil {
140
+
continue
140
141
}
142
+
143
+
langs = append(langs, db.RepoLanguage{
144
+
RepoAt: repo.RepoAt(),
145
+
Ref: ref.Short(),
146
+
Language: l.Lang,
147
+
Bytes: l.Size,
148
+
})
141
149
}
142
150
143
-
return nil
151
+
return db.InsertRepoLanguages(d, langs)
144
152
}
145
153
146
154
func ingestPipeline(d *db.DB, source ec.Source, msg ec.Message) error {