this repo has no description
update transform to match simplified url/domain schema
+22
-10
+4
-2
search/testdata/transform-post-fixtures.json
+4
-2
search/testdata/transform-post-fixtures.json
···
34
34
"record_cid": "bafyreibjifzpqj6o6wcq3hejh7y4z4z2vmiklkvykc57tw3pcbx3kxifpm",
35
35
"created_at": "2023-08-07T05:46:14.423045Z",
36
36
"text": "post which embeds an external URL as a card",
37
-
"embed_url": "https://bsky.app",
37
+
"url": ["https://bsky.app"],
38
+
"domain": ["bsky.app"],
38
39
"embed_img_count": 0
39
40
}
40
41
},
···
123
124
"created_at": "2023-08-07T05:46:14.423045Z",
124
125
"text": "longer example with #some #hashtags, emoji \u2620 \ud83d\ude42 \ud83c\udf85\ud83c\udfff, flags \ud83c\uddf8\ud83c\udde8 ",
125
126
"reply_root_aturi": "at://did:plc:u5cwb2mwiv2bfq53cjufe6yn/app.bsky.feed.post/3k43tv4rft22g",
126
-
"link_url": [ "https://en.wikipedia.org/wiki/CBOR" ],
127
127
"mention_did": [ "did:plc:ewvi7nxzyoun6zhxrhs64oiz" ],
128
128
"embed_aturi": "at://did:plc:u5cwb2mwiv2bfq53cjufe6yn/app.bsky.feed.post/3k44deefqdk2g",
129
129
"lang_code": ["th", "en-US"],
130
130
"lang_code_iso2": ["th", "en"],
131
131
"self_label": ["nudity"],
132
+
"url": [ "https://en.wikipedia.org/wiki/CBOR" ],
133
+
"domain": [ "en.wikipedia.org" ],
132
134
"tag": ["some", "thing"],
133
135
"emoji": ["\u2620", "\ud83d\ude42", "\ud83c\udf85\ud83c\udfff", "\ud83c\uddf8\ud83c\udde8"],
134
136
"embed_img_count": 0
+18
-8
search/transform.go
+18
-8
search/transform.go
···
2
2
3
3
import (
4
4
"log/slog"
5
+
"net/url"
5
6
"strings"
6
7
"time"
7
8
···
21
22
Description *string `json:"description,omitempty"`
22
23
ImgAltText []string `json:"img_alt_text,omitempty"`
23
24
SelfLabel []string `json:"self_label,omitempty"`
25
+
URL []string `json:"url,omitempty"`
26
+
Domain []string `json:"domain,omitempty"`
24
27
Tag []string `json:"tag,omitempty"`
25
28
Emoji []string `json:"emoji,omitempty"`
26
29
HasAvatar bool `json:"has_avatar"`
···
38
41
LangCode []string `json:"lang_code,omitempty"`
39
42
LangCodeIso2 []string `json:"lang_code_iso2,omitempty"`
40
43
MentionDID []string `json:"mention_did,omitempty"`
41
-
LinkURL []string `json:"link_url,omitempty"`
42
-
EmbedURL *string `json:"embed_url,omitempty"`
43
44
EmbedATURI *string `json:"embed_aturi,omitempty"`
44
45
ReplyRootATURI *string `json:"reply_root_aturi,omitempty"`
45
46
EmbedImgCount int `json:"embed_img_count"`
46
47
EmbedImgAltText []string `json:"embed_img_alt_text,omitempty"`
47
48
EmbedImgAltTextJA []string `json:"embed_img_alt_text_ja,omitempty"`
48
49
SelfLabel []string `json:"self_label,omitempty"`
50
+
URL []string `json:"url,omitempty"`
51
+
Domain []string `json:"domain,omitempty"`
49
52
Tag []string `json:"tag,omitempty"`
50
53
Emoji []string `json:"emoji,omitempty"`
51
54
}
···
117
120
}
118
121
}
119
122
var mentionDIDs []string
120
-
var linkURLs []string
123
+
var urls []string
121
124
for _, facet := range post.Facets {
122
125
for _, feat := range facet.Features {
123
126
if feat.RichtextFacet_Mention != nil {
124
127
mentionDIDs = append(mentionDIDs, feat.RichtextFacet_Mention.Did)
125
128
}
126
129
if feat.RichtextFacet_Link != nil {
127
-
linkURLs = append(linkURLs, feat.RichtextFacet_Link.Uri)
130
+
urls = append(urls, feat.RichtextFacet_Link.Uri)
128
131
}
129
132
}
130
133
}
···
132
135
if post.Reply != nil {
133
136
replyRootATURI = &(post.Reply.Root.Uri)
134
137
}
135
-
var embedURL *string
136
138
if post.Embed != nil && post.Embed.EmbedExternal != nil {
137
-
embedURL = &post.Embed.EmbedExternal.External.Uri
139
+
urls = append(urls, post.Embed.EmbedExternal.External.Uri)
138
140
}
139
141
var embedATURI *string
140
142
if post.Embed != nil && post.Embed.EmbedRecord != nil {
···
164
166
}
165
167
}
166
168
169
+
var domains []string
170
+
for _, raw := range urls {
171
+
u, err := url.Parse(raw)
172
+
if nil == err {
173
+
domains = append(domains, u.Hostname())
174
+
}
175
+
}
176
+
167
177
doc := PostDoc{
168
178
DocIndexTs: syntax.DatetimeNow().String(),
169
179
DID: ident.DID.String(),
···
173
183
LangCode: post.Langs,
174
184
LangCodeIso2: langCodeIso2,
175
185
MentionDID: mentionDIDs,
176
-
LinkURL: linkURLs,
177
-
EmbedURL: embedURL,
178
186
EmbedATURI: embedATURI,
179
187
ReplyRootATURI: replyRootATURI,
180
188
EmbedImgCount: embedImgCount,
181
189
EmbedImgAltText: embedImgAltText,
182
190
EmbedImgAltTextJA: embedImgAltTextJA,
183
191
SelfLabel: selfLabels,
192
+
URL: urls,
193
+
Domain: domains,
184
194
Tag: parsePostTags(post),
185
195
Emoji: parseEmojis(post.Text),
186
196
}