this repo has no description
palomar: prod errors burndown (#481)
I took a peek at prod error logs and found some issues:
- deleting search documents is failing, because the "document ID"
(docID) was formatted wrong, because the `rkey` parameter is actually a
full path, not just the record key part
- many actually-valid datetimes were not getting parsed correctly,
resulting in posts not having correct createdAt, likely resulting in
them not appearing with expected ranking in search results
- many HTTP 404 errors on fetches of base path; not really a big deal
but might as well return healthcheck
I haven't tested these much yet, we should probably deploy to staging
(which consumes the prod firehose, IIRC) and observe that for a bit.
+37
-24
+25
-19
search/indexing.go
+25
-19
search/indexing.go
···
6
6
"encoding/json"
7
7
"fmt"
8
8
"io"
9
-
"regexp"
10
9
"strings"
11
10
12
11
appbsky "github.com/bluesky-social/indigo/api/bsky"
···
18
17
esapi "github.com/opensearch-project/opensearch-go/v2/opensearchapi"
19
18
)
20
19
21
-
func (s *Server) deletePost(ctx context.Context, ident *identity.Identity, rkey string) error {
20
+
func (s *Server) deletePost(ctx context.Context, ident *identity.Identity, recordPath string) error {
22
21
ctx, span := tracer.Start(ctx, "deletePost")
23
22
defer span.End()
24
-
span.SetAttributes(attribute.String("repo", ident.DID.String()), attribute.String("rkey", rkey))
23
+
span.SetAttributes(attribute.String("repo", ident.DID.String()), attribute.String("path", recordPath))
25
24
26
-
log := s.logger.With("repo", ident.DID, "rkey", rkey, "op", "deletePost")
27
-
log.Info("deleting post from index")
25
+
logger := s.logger.With("repo", ident.DID, "path", recordPath, "op", "deletePost")
26
+
27
+
parts := strings.SplitN(recordPath, "/", 3)
28
+
if len(parts) < 2 {
29
+
logger.Warn("skipping post record with malformed path")
30
+
return nil
31
+
}
32
+
rkey, err := syntax.ParseTID(parts[1])
33
+
if err != nil {
34
+
logger.Warn("skipping post record with non-TID rkey")
35
+
return nil
36
+
}
37
+
28
38
docID := fmt.Sprintf("%s_%s", ident.DID.String(), rkey)
39
+
logger.Info("deleting post from index", "docID", docID)
29
40
req := esapi.DeleteRequest{
30
41
Index: s.postIndex,
31
42
DocumentID: docID,
···
42
53
return fmt.Errorf("failed to read indexing response: %w", err)
43
54
}
44
55
if res.IsError() {
45
-
log.Warn("opensearch indexing error", "status_code", res.StatusCode, "response", res, "body", string(body))
56
+
logger.Warn("opensearch indexing error", "status_code", res.StatusCode, "response", res, "body", string(body))
46
57
return fmt.Errorf("indexing error, code=%d", res.StatusCode)
47
58
}
48
59
return nil
49
60
}
50
-
51
-
var tidRegex = regexp.MustCompile(`^[234567abcdefghijklmnopqrstuvwxyz]{13}$`)
52
61
53
62
func (s *Server) indexPost(ctx context.Context, ident *identity.Identity, rec *appbsky.FeedPost, path string, rcid cid.Cid) error {
54
63
ctx, span := tracer.Start(ctx, "indexPost")
···
57
66
58
67
log := s.logger.With("repo", ident.DID, "path", path, "op", "indexPost")
59
68
parts := strings.SplitN(path, "/", 3)
60
-
// TODO: replace with an atproto/syntax package type for TID
61
-
if len(parts) != 2 || !tidRegex.MatchString(parts[1]) {
62
-
log.Warn("skipping index post record with weird path/TID", "did", ident.DID, "path", path)
69
+
if len(parts) < 2 {
70
+
log.Warn("skipping post record with malformed path")
63
71
return nil
64
72
}
65
-
rkey := parts[1]
66
-
67
-
log = log.With("rkey", rkey)
68
-
69
-
_, err := syntax.ParseDatetimeLenient(rec.CreatedAt)
73
+
rkey, err := syntax.ParseTID(parts[1])
70
74
if err != nil {
71
-
log.Warn("post had invalid timestamp", "createdAt", rec.CreatedAt, "parseErr", err)
72
-
rec.CreatedAt = ""
75
+
log.Warn("skipping post record with non-TID rkey")
76
+
return nil
73
77
}
74
78
75
-
doc := TransformPost(rec, ident, rkey, rcid.String())
79
+
log = log.With("rkey", rkey)
80
+
81
+
doc := TransformPost(rec, ident, rkey.String(), rcid.String())
76
82
b, err := json.Marshal(doc)
77
83
if err != nil {
78
84
return err
+2
search/server.go
+2
search/server.go
···
168
168
}
169
169
170
170
type HealthStatus struct {
171
+
Service string `json:"service,const=palomar"`
171
172
Status string `json:"status"`
172
173
Version string `json:"version"`
173
174
Message string `json:"msg,omitempty"`
···
202
203
}
203
204
204
205
e.Use(middleware.CORS())
206
+
e.GET("/", s.handleHealthCheck)
205
207
e.GET("/_health", s.handleHealthCheck)
206
208
e.GET("/metrics", echo.WrapHandler(promhttp.Handler()))
207
209
e.GET("/xrpc/app.bsky.unspecced.searchPostsSkeleton", s.handleSearchPostsSkeleton)
+10
-5
search/transform.go
+10
-5
search/transform.go
···
2
2
3
3
import (
4
4
"strings"
5
-
"time"
6
5
7
6
appbsky "github.com/bluesky-social/indigo/api/bsky"
8
7
"github.com/bluesky-social/indigo/atproto/identity"
9
-
"github.com/bluesky-social/indigo/util"
8
+
"github.com/bluesky-social/indigo/atproto/syntax"
9
+
10
10
"github.com/rivo/uniseg"
11
11
)
12
12
···
80
80
handle = ident.Handle.String()
81
81
}
82
82
return ProfileDoc{
83
-
DocIndexTs: time.Now().UTC().Format(util.ISO8601),
83
+
DocIndexTs: syntax.DatetimeNow().String(),
84
84
DID: ident.DID.String(),
85
85
RecordCID: cid,
86
86
Handle: handle,
···
157
157
}
158
158
159
159
doc := PostDoc{
160
-
DocIndexTs: time.Now().UTC().Format(util.ISO8601),
160
+
DocIndexTs: syntax.DatetimeNow().String(),
161
161
DID: ident.DID.String(),
162
162
RecordRkey: rkey,
163
163
RecordCID: cid,
···
177
177
}
178
178
179
179
if post.CreatedAt != "" {
180
-
doc.CreatedAt = &post.CreatedAt
180
+
// there are some old bad timestamps out there!
181
+
dt, err := syntax.ParseDatetimeLenient(post.CreatedAt)
182
+
if nil == err { // *not* an error
183
+
s := dt.String()
184
+
doc.CreatedAt = &s
185
+
}
181
186
}
182
187
183
188
return doc