this repo has no description
feat: start work on feedweb
+208
-8
+7
Makefile
+7
Makefile
+61
cmd/feedweb/feedweb.go
+61
cmd/feedweb/feedweb.go
···
1
+
package main
2
+
3
+
import (
4
+
"net/http"
5
+
6
+
"github.com/labstack/echo/v4"
7
+
"github.com/labstack/echo/v4/middleware"
8
+
"github.com/edavis/bsky-feeds/mostliked"
9
+
)
10
+
11
+
type SkeletonRequest struct {
12
+
Feed string `query:"feed"`
13
+
Limit int64 `query:"limit"`
14
+
Offset string `query:"offset"`
15
+
}
16
+
17
+
type SkeletonResponse struct {
18
+
Cursor string `json:"cursor,omitempty"`
19
+
Feed []Post `json:"feed"`
20
+
}
21
+
22
+
type Post struct {
23
+
Uri string `json:"post"`
24
+
}
25
+
26
+
type SkeletonHeader struct {
27
+
Langs []string `header:"Accept-Language"`
28
+
}
29
+
30
+
func getFeedSkeleton(c echo.Context) error {
31
+
var req SkeletonRequest
32
+
if err := c.Bind(&req); err != nil {
33
+
return c.String(http.StatusBadRequest, "bad request")
34
+
}
35
+
var hdr SkeletonHeader
36
+
if err := (&echo.DefaultBinder{}).BindHeaders(c, &hdr); err != nil {
37
+
return c.String(http.StatusBadRequest, "bad request")
38
+
}
39
+
40
+
var posts []Post
41
+
uris := mostliked.Feed(mostliked.FeedViewParams{
42
+
Limit: req.Limit,
43
+
Offset: req.Offset,
44
+
Langs: hdr.Langs,
45
+
})
46
+
for _, uri := range uris {
47
+
posts = append(posts, Post{uri})
48
+
}
49
+
response := SkeletonResponse{
50
+
Feed: posts,
51
+
}
52
+
return c.JSON(http.StatusOK, response)
53
+
}
54
+
55
+
func main() {
56
+
e := echo.New()
57
+
e.Use(middleware.Logger())
58
+
e.Use(middleware.Recover())
59
+
e.GET("/xrpc/app.bsky.feed.getFeedSkeleton", getFeedSkeleton)
60
+
e.Logger.Fatal(e.Start(":5000"))
61
+
}
+8
-8
cmd/mostliked/mostliked.go
+8
-8
cmd/mostliked/mostliked.go
···
12
12
13
13
appbsky "github.com/bluesky-social/indigo/api/bsky"
14
14
jetstream "github.com/bluesky-social/jetstream/pkg/models"
15
-
"github.com/edavis/bsky-feeds/pkg/mostliked"
15
+
db "github.com/edavis/bsky-feeds/pkg/mostliked"
16
16
"github.com/gorilla/websocket"
17
17
"github.com/karlseguin/ccache/v3"
18
18
_ "github.com/mattn/go-sqlite3"
···
60
60
}
61
61
}
62
62
63
-
func trimPostsTable(ctx context.Context, queries *mostliked.Queries) {
63
+
func trimPostsTable(ctx context.Context, queries *db.Queries) {
64
64
ticker := time.NewTicker(1 * time.Minute)
65
65
defer ticker.Stop()
66
66
···
94
94
func processEvents(events <-chan []byte) {
95
95
ctx := context.Background()
96
96
97
-
db, err := sql.Open("sqlite3", "db/mostliked.db?_journal=WAL&_fk=on")
97
+
dbCnx, err := sql.Open("sqlite3", "db/mostliked.db?_journal=WAL&_fk=on")
98
98
if err != nil {
99
99
log.Fatal("error opening db")
100
100
}
101
-
if _, err := db.ExecContext(ctx, ddl); err != nil {
101
+
if _, err := dbCnx.ExecContext(ctx, ddl); err != nil {
102
102
log.Fatal("couldn't create tables")
103
103
}
104
-
defer db.Close()
104
+
defer dbCnx.Close()
105
105
106
-
queries := mostliked.New(db)
106
+
queries := db.New(dbCnx)
107
107
108
108
drafts := ccache.New(ccache.Configure[DraftPost]().MaxSize(50_000).GetsPerPromote(1))
109
109
···
174
174
}
175
175
drafts.Delete(like.Subject.Uri)
176
176
log.Println("storing", like.Subject.Uri, "in database")
177
-
err := queries.InsertPost(ctx, mostliked.InsertPostParams{
177
+
err := queries.InsertPost(ctx, db.InsertPostParams{
178
178
Uri: like.Subject.Uri,
179
179
CreateTs: draftPost.Created,
180
180
Likes: draftPost.Likes,
···
182
182
if err != nil {
183
183
log.Println("error inserting post")
184
184
}
185
-
err = queries.InsertLang(ctx, mostliked.InsertLangParams{
185
+
err = queries.InsertLang(ctx, db.InsertLangParams{
186
186
Uri: like.Subject.Uri,
187
187
Lang: strings.ToLower(draftPost.Language.IsoCode639_1().String()),
188
188
})
+7
cmd/mostliked/query.sql
+7
cmd/mostliked/query.sql
···
9
9
10
10
-- name: TrimPosts :exec
11
11
delete from posts where create_ts < unixepoch('now', '-24 hours');
12
+
13
+
-- name: ViewFeed :many
14
+
select posts.uri, create_ts, likes, lang
15
+
from posts
16
+
left join langs on posts.uri = langs.uri
17
+
order by likes desc
18
+
limit ? offset ?;
+9
go.mod
+9
go.mod
···
7
7
github.com/bluesky-social/jetstream v0.0.0-20240925210745-2cd1b6147279
8
8
github.com/gorilla/websocket v1.5.3
9
9
github.com/karlseguin/ccache/v3 v3.0.5
10
+
github.com/labstack/echo/v4 v4.11.3
10
11
github.com/mattn/go-sqlite3 v1.14.22
11
12
github.com/pemistahl/lingua-go v1.4.0
12
13
)
···
18
19
github.com/go-logr/stdr v1.2.2 // indirect
19
20
github.com/goccy/go-json v0.10.2 // indirect
20
21
github.com/gogo/protobuf v1.3.2 // indirect
22
+
github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
21
23
github.com/google/uuid v1.6.0 // indirect
22
24
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
23
25
github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
···
36
38
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
37
39
github.com/jbenet/goprocess v0.1.4 // indirect
38
40
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
41
+
github.com/labstack/gommon v0.4.1 // indirect
42
+
github.com/mattn/go-colorable v0.1.13 // indirect
39
43
github.com/mattn/go-isatty v0.0.20 // indirect
40
44
github.com/minio/sha256-simd v1.0.1 // indirect
41
45
github.com/mr-tron/base58 v1.2.0 // indirect
···
48
52
github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect
49
53
github.com/shopspring/decimal v1.3.1 // indirect
50
54
github.com/spaolacci/murmur3 v1.1.0 // indirect
55
+
github.com/valyala/bytebufferpool v1.0.0 // indirect
56
+
github.com/valyala/fasttemplate v1.2.2 // indirect
51
57
github.com/whyrusleeping/cbor-gen v0.1.3-0.20240904181319-8dc02b38228c // indirect
52
58
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect
53
59
go.opentelemetry.io/otel v1.21.0 // indirect
···
58
64
go.uber.org/zap v1.26.0 // indirect
59
65
golang.org/x/crypto v0.22.0 // indirect
60
66
golang.org/x/exp v0.0.0-20240604190554-fc45aab8b7f8 // indirect
67
+
golang.org/x/net v0.24.0 // indirect
61
68
golang.org/x/sys v0.22.0 // indirect
69
+
golang.org/x/text v0.16.0 // indirect
70
+
golang.org/x/time v0.5.0 // indirect
62
71
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
63
72
google.golang.org/protobuf v1.34.2 // indirect
64
73
lukechampine.com/blake3 v1.2.1 // indirect
+20
go.sum
+20
go.sum
···
22
22
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
23
23
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
24
24
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
25
+
github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
26
+
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
25
27
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
26
28
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
27
29
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
···
84
86
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
85
87
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
86
88
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
89
+
github.com/labstack/echo/v4 v4.11.3 h1:Upyu3olaqSHkCjs1EJJwQ3WId8b8b1hxbogyommKktM=
90
+
github.com/labstack/echo/v4 v4.11.3/go.mod h1:UcGuQ8V6ZNRmSweBIJkPvGfwCMIlFmiqrPqiEBfPYws=
91
+
github.com/labstack/gommon v0.4.1 h1:gqEff0p/hTENGMABzezPoPSRtIh1Cvw0ueMOe0/dfOk=
92
+
github.com/labstack/gommon v0.4.1/go.mod h1:TyTrpPqxR5KMk8LKVtLmfMjeQ5FEkBYdxLYPw/WfrOM=
93
+
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
94
+
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
87
95
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
96
+
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
88
97
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
89
98
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
90
99
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
···
133
142
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
134
143
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
135
144
github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
145
+
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
146
+
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
147
+
github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo=
148
+
github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
136
149
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=
137
150
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
138
151
github.com/whyrusleeping/cbor-gen v0.1.3-0.20240904181319-8dc02b38228c h1:UsxJNcLPfyLyVaA4iusIrsLAqJn/xh36Qgb8emqtXzk=
···
183
196
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
184
197
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
185
198
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
199
+
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
200
+
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
186
201
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
187
202
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
188
203
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
···
194
209
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
195
210
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
196
211
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
212
+
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
197
213
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
198
214
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
199
215
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
···
201
217
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
202
218
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
203
219
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
220
+
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
221
+
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
222
+
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
223
+
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
204
224
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
205
225
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
206
226
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+45
mostliked/feed.go
+45
mostliked/feed.go
···
1
+
package mostliked
2
+
3
+
import (
4
+
"context"
5
+
"database/sql"
6
+
"log"
7
+
"strconv"
8
+
9
+
db "github.com/edavis/bsky-feeds/pkg/mostliked"
10
+
_ "github.com/mattn/go-sqlite3"
11
+
)
12
+
13
+
type FeedViewParams struct {
14
+
Limit int64
15
+
Offset string
16
+
Langs []string
17
+
}
18
+
19
+
func Feed(args FeedViewParams) []string {
20
+
ctx := context.Background()
21
+
dbCnx, err := sql.Open("sqlite3", "db/mostliked.db?_journal=WAL&_fk=on")
22
+
if err != nil {
23
+
log.Fatal("error opening db")
24
+
}
25
+
defer dbCnx.Close()
26
+
27
+
offset, err := strconv.Atoi(args.Offset)
28
+
if err != nil {
29
+
log.Println("error converting offset to integer")
30
+
}
31
+
32
+
queries := db.New(dbCnx)
33
+
rows, err := queries.ViewFeed(ctx, db.ViewFeedParams{
34
+
Limit: args.Limit,
35
+
Offset: int64(offset),
36
+
})
37
+
if err != nil {
38
+
log.Println("error fetching rows")
39
+
}
40
+
var uris []string
41
+
for _, row := range rows {
42
+
uris = append(uris, row.Uri)
43
+
}
44
+
return uris
45
+
}
+49
pkg/mostliked/query.sql.go
+49
pkg/mostliked/query.sql.go
···
7
7
8
8
import (
9
9
"context"
10
+
"database/sql"
10
11
)
11
12
12
13
const insertLang = `-- name: InsertLang :exec
···
55
56
_, err := q.db.ExecContext(ctx, updateLikes, uri)
56
57
return err
57
58
}
59
+
60
+
const viewFeed = `-- name: ViewFeed :many
61
+
select posts.uri, create_ts, likes, lang
62
+
from posts
63
+
left join langs on posts.uri = langs.uri
64
+
order by likes desc
65
+
limit ? offset ?
66
+
`
67
+
68
+
type ViewFeedParams struct {
69
+
Limit int64
70
+
Offset int64
71
+
}
72
+
73
+
type ViewFeedRow struct {
74
+
Uri string
75
+
CreateTs int64
76
+
Likes int64
77
+
Lang sql.NullString
78
+
}
79
+
80
+
func (q *Queries) ViewFeed(ctx context.Context, arg ViewFeedParams) ([]ViewFeedRow, error) {
81
+
rows, err := q.db.QueryContext(ctx, viewFeed, arg.Limit, arg.Offset)
82
+
if err != nil {
83
+
return nil, err
84
+
}
85
+
defer rows.Close()
86
+
var items []ViewFeedRow
87
+
for rows.Next() {
88
+
var i ViewFeedRow
89
+
if err := rows.Scan(
90
+
&i.Uri,
91
+
&i.CreateTs,
92
+
&i.Likes,
93
+
&i.Lang,
94
+
); err != nil {
95
+
return nil, err
96
+
}
97
+
items = append(items, i)
98
+
}
99
+
if err := rows.Close(); err != nil {
100
+
return nil, err
101
+
}
102
+
if err := rows.Err(); err != nil {
103
+
return nil, err
104
+
}
105
+
return items, nil
106
+
}