Auto-indexing service and GraphQL API for AT Protocol Records
update styles, add records sparkline, use real-time server component for backfill btn
+922
-199
+85
server/src/backfill_state.gleam
+85
server/src/backfill_state.gleam
···
1
+
/// Global state management for backfill operations.
2
+
///
3
+
/// This module provides a singleton OTP actor that tracks whether a backfill
4
+
/// operation is currently running. The state persists across WebSocket
5
+
/// reconnections and page refreshes, allowing the UI to show accurate backfill
6
+
/// status even if the user refreshes the page during a long-running backfill.
7
+
///
8
+
/// ## Architecture
9
+
///
10
+
/// - Single global actor instance started at server boot
11
+
/// - State is shared across all client connections
12
+
/// - Backfill process updates state when starting/stopping
13
+
/// - UI components poll state to update their displays
14
+
///
15
+
/// ## Example Usage
16
+
///
17
+
/// ```gleam
18
+
/// // Start the actor (done once at server startup)
19
+
/// let assert Ok(backfill_state) = backfill_state.start()
20
+
///
21
+
/// // Start a backfill operation
22
+
/// process.send(backfill_state, backfill_state.StartBackfill)
23
+
///
24
+
/// // Query current state
25
+
/// let is_backfilling = actor.call(
26
+
/// backfill_state,
27
+
/// waiting: 100,
28
+
/// sending: backfill_state.IsBackfilling,
29
+
/// )
30
+
///
31
+
/// // Stop the backfill
32
+
/// process.send(backfill_state, backfill_state.StopBackfill)
33
+
/// ```
34
+
import gleam/erlang/process
35
+
import gleam/otp/actor
36
+
37
+
/// Internal state of the backfill actor
38
+
pub type State {
39
+
State(backfilling: Bool)
40
+
}
41
+
42
+
/// Messages that can be sent to the backfill state actor
43
+
pub type Message {
44
+
/// Query whether a backfill is currently running.
45
+
/// The actor will send the current state to the provided Subject.
46
+
IsBackfilling(reply_with: process.Subject(Bool))
47
+
/// Set backfilling state to True (sent when backfill begins)
48
+
StartBackfill
49
+
/// Set backfilling state to False (sent when backfill completes)
50
+
StopBackfill
51
+
}
52
+
53
+
/// Start the backfill state actor.
54
+
///
55
+
/// This should be called once during server initialization.
56
+
/// Returns a Subject that can be used to send messages to the actor.
57
+
pub fn start() -> Result(process.Subject(Message), actor.StartError) {
58
+
let result =
59
+
actor.new(State(backfilling: False))
60
+
|> actor.on_message(handle_message)
61
+
|> actor.start
62
+
63
+
case result {
64
+
Ok(started) -> Ok(started.data)
65
+
Error(err) -> Error(err)
66
+
}
67
+
}
68
+
69
+
/// Handle incoming messages to update or query the backfill state
70
+
fn handle_message(state: State, message: Message) -> actor.Next(State, Message) {
71
+
case message {
72
+
IsBackfilling(client) -> {
73
+
process.send(client, state.backfilling)
74
+
actor.continue(state)
75
+
}
76
+
77
+
StartBackfill -> {
78
+
actor.continue(State(backfilling: True))
79
+
}
80
+
81
+
StopBackfill -> {
82
+
actor.continue(State(backfilling: False))
83
+
}
84
+
}
85
+
}
+14
-14
server/src/components/collection_table.gleam
+14
-14
server/src/components/collection_table.gleam
···
1
1
import database
2
-
import gleam/int
2
+
import format
3
3
import gleam/list
4
4
import lustre/attribute
5
5
import lustre/element.{type Element}
···
15
15
html.div(
16
16
[
17
17
attribute.class(
18
-
"bg-white rounded-lg shadow-sm border border-gray-200 overflow-hidden",
18
+
"bg-zinc-900 rounded-lg shadow-sm border border-zinc-800 overflow-hidden",
19
19
),
20
20
],
21
21
[
22
-
html.table([attribute.class("min-w-full divide-y divide-gray-200")], [
22
+
html.table([attribute.class("min-w-full divide-y divide-zinc-800")], [
23
23
render_header(),
24
-
html.tbody([attribute.class("bg-white divide-y divide-gray-200")], rows),
24
+
html.tbody([attribute.class("bg-zinc-900 divide-y divide-zinc-800")], rows),
25
25
]),
26
26
],
27
27
)
···
29
29
30
30
/// Render the table header
31
31
fn render_header() -> Element(msg) {
32
-
html.thead([attribute.class("bg-gray-50")], [
32
+
html.thead([attribute.class("bg-zinc-900")], [
33
33
html.tr([], [
34
34
html.th(
35
35
[
36
36
attribute.class(
37
-
"px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider",
37
+
"px-4 py-3 text-left text-xs font-medium text-zinc-500 uppercase tracking-wider",
38
38
),
39
39
],
40
40
[element.text("Collection")],
···
42
42
html.th(
43
43
[
44
44
attribute.class(
45
-
"px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider",
45
+
"px-4 py-3 text-left text-xs font-medium text-zinc-500 uppercase tracking-wider",
46
46
),
47
47
],
48
48
[element.text("Record Count")],
···
76
76
77
77
/// Render a row for a collection with records
78
78
fn render_stat_row(collection: String, count: Int) -> Element(msg) {
79
-
html.tr([attribute.class("hover:bg-gray-50 transition-colors")], [
80
-
html.td([attribute.class("px-4 py-3 text-sm text-gray-900")], [
79
+
html.tr([attribute.class("hover:bg-zinc-800 transition-colors")], [
80
+
html.td([attribute.class("px-4 py-3 text-sm text-zinc-200")], [
81
81
element.text(collection),
82
82
]),
83
-
html.td([attribute.class("px-4 py-3 text-sm text-gray-700")], [
84
-
element.text(int.to_string(count)),
83
+
html.td([attribute.class("px-4 py-3 text-sm text-zinc-300")], [
84
+
element.text(format.format_number(count)),
85
85
]),
86
86
])
87
87
}
88
88
89
89
/// Render a row for a lexicon without records yet
90
90
fn render_empty_row(collection: String) -> Element(msg) {
91
-
html.tr([attribute.class("hover:bg-gray-50 transition-colors")], [
92
-
html.td([attribute.class("px-4 py-3 text-sm text-gray-900")], [
91
+
html.tr([attribute.class("hover:bg-zinc-800 transition-colors")], [
92
+
html.td([attribute.class("px-4 py-3 text-sm text-zinc-200")], [
93
93
element.text(collection),
94
94
]),
95
-
html.td([attribute.class("px-4 py-3 text-sm text-gray-500 italic")], [
95
+
html.td([attribute.class("px-4 py-3 text-sm text-zinc-500 italic")], [
96
96
element.text("0"),
97
97
]),
98
98
])
+26
-2
server/src/components/layout.gleam
+26
-2
server/src/components/layout.gleam
···
28
28
[attribute.attribute("src", "https://cdn.tailwindcss.com")],
29
29
[],
30
30
),
31
+
// Lustre server component runtime
32
+
html.script(
33
+
[
34
+
attribute.type_("module"),
35
+
attribute.attribute("src", "/lustre/runtime.mjs"),
36
+
],
37
+
"",
38
+
),
39
+
// Listen for backfill-complete event and reload page
40
+
html.script(
41
+
[],
42
+
"
43
+
// Wait for DOM to be ready
44
+
document.addEventListener('DOMContentLoaded', function() {
45
+
const backfillButton = document.querySelector('lustre-server-component#backfill-button');
46
+
if (backfillButton) {
47
+
backfillButton.addEventListener('backfill-complete', function() {
48
+
// Reload page to show updated database stats
49
+
window.location.reload();
50
+
});
51
+
}
52
+
});
53
+
",
54
+
),
31
55
])
32
56
}
33
57
34
58
/// Renders the HTML body with a max-width container
35
59
fn body(content: List(Element(msg))) -> Element(msg) {
36
-
html.body([attribute.class("bg-gray-50 min-h-screen p-8")], [
37
-
html.div([attribute.class("max-w-4xl mx-auto")], content),
60
+
html.body([attribute.class("bg-zinc-950 text-zinc-300 font-mono min-h-screen")], [
61
+
html.div([attribute.class("max-w-4xl mx-auto px-6 py-12")], content),
38
62
])
39
63
}
+135
server/src/components/sparkline.gleam
+135
server/src/components/sparkline.gleam
···
1
+
import database
2
+
import gleam/float
3
+
import gleam/int
4
+
import gleam/list
5
+
import gleam/string
6
+
import lustre/attribute
7
+
import lustre/element.{type Element}
8
+
9
+
/// Renders a sparkline chart from activity data points
10
+
pub fn view(activity: List(database.ActivityPoint)) -> Element(msg) {
11
+
case activity {
12
+
[] -> element.none()
13
+
_ -> render_chart(activity)
14
+
}
15
+
}
16
+
17
+
fn render_chart(data: List(database.ActivityPoint)) -> Element(msg) {
18
+
let width = 800
19
+
let height = 80
20
+
21
+
// Calculate min/max for scaling
22
+
let counts = list.map(data, fn(point) { point.count })
23
+
let max = case list.reduce(counts, int.max) {
24
+
Ok(m) -> int.max(m, 1)
25
+
Error(_) -> 1
26
+
}
27
+
let min = case list.reduce(counts, int.min) {
28
+
Ok(m) -> int.min(m, 0)
29
+
Error(_) -> 0
30
+
}
31
+
32
+
let range = max - min
33
+
let range_float = case range {
34
+
0 -> 1.0
35
+
_ -> int.to_float(range)
36
+
}
37
+
let max_height = int.to_float(height) *. 0.75
38
+
39
+
// Generate polyline points
40
+
let data_length = list.length(data)
41
+
let points_string =
42
+
data
43
+
|> list.index_map(fn(point, index) {
44
+
let x = case data_length {
45
+
1 -> int.to_float(width) /. 2.0
46
+
_ -> int.to_float(index) /. int.to_float(data_length - 1) *. int.to_float(width)
47
+
}
48
+
let y_normalized = int.to_float(point.count - min) /. range_float
49
+
let y = int.to_float(height) -. y_normalized *. max_height
50
+
51
+
float.to_string(x) <> "," <> float.to_string(y)
52
+
})
53
+
|> string.join(" ")
54
+
55
+
// Generate area path for gradient fill
56
+
let area_path =
57
+
"M 0," <> int.to_string(height)
58
+
<> " L " <> points_string
59
+
<> " L " <> int.to_string(width) <> "," <> int.to_string(height)
60
+
<> " Z"
61
+
62
+
// Create SVG element
63
+
element.element(
64
+
"svg",
65
+
[
66
+
attribute.attribute("width", int.to_string(width)),
67
+
attribute.attribute("height", int.to_string(height)),
68
+
attribute.class("w-full"),
69
+
attribute.attribute("viewBox", "0 0 " <> int.to_string(width) <> " " <> int.to_string(height)),
70
+
attribute.attribute("preserveAspectRatio", "none"),
71
+
],
72
+
[
73
+
// Define gradient
74
+
element.element(
75
+
"defs",
76
+
[],
77
+
[
78
+
element.element(
79
+
"linearGradient",
80
+
[
81
+
attribute.id("sparklineGradient"),
82
+
attribute.attribute("x1", "0%"),
83
+
attribute.attribute("y1", "0%"),
84
+
attribute.attribute("x2", "0%"),
85
+
attribute.attribute("y2", "100%"),
86
+
],
87
+
[
88
+
element.element(
89
+
"stop",
90
+
[
91
+
attribute.attribute("offset", "0%"),
92
+
attribute.attribute("stop-color", "#22d3ee"),
93
+
attribute.attribute("stop-opacity", "0.5"),
94
+
],
95
+
[],
96
+
),
97
+
element.element(
98
+
"stop",
99
+
[
100
+
attribute.attribute("offset", "100%"),
101
+
attribute.attribute("stop-color", "#22d3ee"),
102
+
attribute.attribute("stop-opacity", "0.1"),
103
+
],
104
+
[],
105
+
),
106
+
],
107
+
),
108
+
],
109
+
),
110
+
// Area fill
111
+
element.element(
112
+
"path",
113
+
[
114
+
attribute.attribute("d", area_path),
115
+
attribute.attribute("fill", "url(#sparklineGradient)"),
116
+
attribute.attribute("stroke-width", "0"),
117
+
],
118
+
[],
119
+
),
120
+
// Line
121
+
element.element(
122
+
"polyline",
123
+
[
124
+
attribute.attribute("points", points_string),
125
+
attribute.attribute("fill", "none"),
126
+
attribute.attribute("stroke", "#22d3ee"),
127
+
attribute.attribute("stroke-width", "2"),
128
+
attribute.attribute("stroke-linecap", "round"),
129
+
attribute.attribute("stroke-linejoin", "round"),
130
+
],
131
+
[],
132
+
),
133
+
],
134
+
)
135
+
}
+5
-5
server/src/components/stats_card.gleam
+5
-5
server/src/components/stats_card.gleam
···
9
9
description description: String,
10
10
color color: String,
11
11
) -> Element(msg) {
12
-
let bg_class = "bg-" <> color <> "-50"
13
-
let border_class = "border-" <> color <> "-100"
14
-
let text_class = "text-" <> color <> "-600"
12
+
let bg_class = "bg-" <> color <> "-900/20"
13
+
let border_class = "border-" <> color <> "-800"
14
+
let text_class = "text-" <> color <> "-400"
15
15
16
16
html.div(
17
17
[
18
18
attribute.class(
19
-
bg_class <> " rounded-lg p-6 border " <> border_class <> " shadow-sm",
19
+
"bg-zinc-900 " <> bg_class <> " rounded-lg p-6 border " <> border_class <> " shadow-sm",
20
20
),
21
21
],
22
22
[
23
23
html.div([attribute.class("text-4xl font-bold " <> text_class <> " mb-2")], [
24
24
element.text(int.to_string(count)),
25
25
]),
26
-
html.div([attribute.class("text-gray-600")], [element.text(description)]),
26
+
html.div([attribute.class("text-zinc-400")], [element.text(description)]),
27
27
],
28
28
)
29
29
}
+61
server/src/database.gleam
+61
server/src/database.gleam
···
606
606
}
607
607
}
608
608
609
+
/// Gets the total number of records in the database
610
+
pub fn get_record_count(conn: sqlight.Connection) -> Result(Int, sqlight.Error) {
611
+
let sql =
612
+
"
613
+
SELECT COUNT(*) as count
614
+
FROM record
615
+
"
616
+
617
+
let decoder = {
618
+
use count <- decode.field(0, decode.int)
619
+
decode.success(count)
620
+
}
621
+
622
+
case sqlight.query(sql, on: conn, with: [], expecting: decoder) {
623
+
Ok([count]) -> Ok(count)
624
+
Ok(_) -> Ok(0)
625
+
Error(err) -> Error(err)
626
+
}
627
+
}
628
+
609
629
/// Checks if a lexicon exists for a given collection NSID
610
630
/// First checks the dedicated lexicon table, then falls back to record table
611
631
pub fn has_lexicon_for_collection(
···
780
800
use json <- decode.field(1, decode.string)
781
801
use created_at <- decode.field(2, decode.string)
782
802
decode.success(Lexicon(id:, json:, created_at:))
803
+
}
804
+
805
+
sqlight.query(sql, on: conn, with: [], expecting: decoder)
806
+
}
807
+
808
+
pub type ActivityPoint {
809
+
ActivityPoint(timestamp: String, count: Int)
810
+
}
811
+
812
+
/// Gets record indexing activity over time
813
+
/// Returns hourly counts for the specified duration
814
+
pub fn get_record_activity(
815
+
conn: sqlight.Connection,
816
+
duration_hours: Int,
817
+
) -> Result(List(ActivityPoint), sqlight.Error) {
818
+
// SQLite datetime calculation for cutoff time
819
+
let sql =
820
+
"
821
+
WITH RECURSIVE time_series AS (
822
+
SELECT datetime('now', '-" <> int.to_string(duration_hours) <> " hours') AS bucket
823
+
UNION ALL
824
+
SELECT datetime(bucket, '+1 hour')
825
+
FROM time_series
826
+
WHERE bucket < datetime('now')
827
+
)
828
+
SELECT
829
+
strftime('%Y-%m-%dT%H:00:00Z', ts.bucket) as timestamp,
830
+
COALESCE(COUNT(r.uri), 0) as count
831
+
FROM time_series ts
832
+
LEFT JOIN record r ON
833
+
datetime(r.indexed_at) >= datetime(ts.bucket)
834
+
AND datetime(r.indexed_at) < datetime(ts.bucket, '+1 hour')
835
+
AND datetime(r.indexed_at) >= datetime('now', '-" <> int.to_string(duration_hours) <> " hours')
836
+
GROUP BY ts.bucket
837
+
ORDER BY ts.bucket ASC
838
+
"
839
+
840
+
let decoder = {
841
+
use timestamp <- decode.field(0, decode.string)
842
+
use count <- decode.field(1, decode.int)
843
+
decode.success(ActivityPoint(timestamp:, count:))
783
844
}
784
845
785
846
sqlight.query(sql, on: conn, with: [], expecting: decoder)
+42
server/src/format.gleam
+42
server/src/format.gleam
···
1
+
import gleam/int
2
+
import gleam/list
3
+
import gleam/string
4
+
5
+
/// Formats an integer with comma thousand separators
6
+
/// Example: format_number(1234567) -> "1,234,567"
7
+
pub fn format_number(num: Int) -> String {
8
+
let str = int.to_string(num)
9
+
let is_negative = string.starts_with(str, "-")
10
+
let digits = case is_negative {
11
+
True -> string.drop_start(str, 1)
12
+
False -> str
13
+
}
14
+
15
+
let chars = string.to_graphemes(digits)
16
+
let char_count = list.length(chars)
17
+
18
+
// If 3 or fewer digits, no commas needed
19
+
case char_count <= 3 {
20
+
True -> str
21
+
False -> {
22
+
let reversed = list.reverse(chars)
23
+
24
+
let formatted =
25
+
reversed
26
+
|> list.index_map(fn(char, idx) {
27
+
case idx > 0 && idx % 3 == 0 {
28
+
True -> [",", char]
29
+
False -> [char]
30
+
}
31
+
})
32
+
|> list.flatten
33
+
|> list.reverse
34
+
|> string.join("")
35
+
36
+
case is_negative {
37
+
True -> "-" <> formatted
38
+
False -> formatted
39
+
}
40
+
}
41
+
}
42
+
}
+129
server/src/lustre_handlers.gleam
+129
server/src/lustre_handlers.gleam
···
1
+
/// WebSocket handlers and setup for Lustre server components.
2
+
///
3
+
/// This module contains all the WebSocket lifecycle handlers and routing
4
+
/// for Lustre server components, including serving the client runtime and
5
+
/// managing component WebSocket connections.
6
+
import backfill_state
7
+
import components/backfill_button
8
+
import gleam/bytes_tree
9
+
import gleam/erlang/application
10
+
import gleam/erlang/process
11
+
import gleam/http/request
12
+
import gleam/http/response
13
+
import gleam/json
14
+
import gleam/option
15
+
import gleam/otp/actor
16
+
import lustre
17
+
import lustre/server_component
18
+
import mist
19
+
import sqlight
20
+
21
+
// LUSTRE RUNTIME
22
+
23
+
/// Serve the Lustre client runtime JavaScript
24
+
pub fn serve_lustre_runtime() -> response.Response(mist.ResponseData) {
25
+
let assert Ok(lustre_priv) = application.priv_directory("lustre")
26
+
let file_path = lustre_priv <> "/static/lustre-server-component.mjs"
27
+
28
+
case mist.send_file(file_path, offset: 0, limit: option.None) {
29
+
Ok(file) ->
30
+
response.new(200)
31
+
|> response.prepend_header("content-type", "application/javascript")
32
+
|> response.set_body(file)
33
+
34
+
Error(_) ->
35
+
response.new(404)
36
+
|> response.set_body(mist.Bytes(bytes_tree.new()))
37
+
}
38
+
}
39
+
40
+
// BACKFILL BUTTON COMPONENT
41
+
42
+
/// WebSocket handler for backfill button component
43
+
pub fn serve_backfill_button(
44
+
req: request.Request(mist.Connection),
45
+
db: sqlight.Connection,
46
+
backfill_state_subject: process.Subject(backfill_state.Message),
47
+
) -> response.Response(mist.ResponseData) {
48
+
mist.websocket(
49
+
request: req,
50
+
on_init: init_backfill_button_socket(db, backfill_state_subject, _),
51
+
handler: loop_backfill_button_socket,
52
+
on_close: close_backfill_button_socket,
53
+
)
54
+
}
55
+
56
+
type BackfillButtonSocket {
57
+
BackfillButtonSocket(
58
+
component: lustre.Runtime(backfill_button.Msg),
59
+
self: process.Subject(server_component.ClientMessage(backfill_button.Msg)),
60
+
)
61
+
}
62
+
63
+
type BackfillButtonSocketMessage =
64
+
server_component.ClientMessage(backfill_button.Msg)
65
+
66
+
type BackfillButtonSocketInit =
67
+
#(BackfillButtonSocket, option.Option(process.Selector(BackfillButtonSocketMessage)))
68
+
69
+
fn init_backfill_button_socket(
70
+
db: sqlight.Connection,
71
+
backfill_state_subject: process.Subject(backfill_state.Message),
72
+
_connection: mist.WebsocketConnection,
73
+
) -> BackfillButtonSocketInit {
74
+
// TODO: Get is_admin from session
75
+
let is_admin = True
76
+
77
+
// Query current backfill state
78
+
let backfilling = actor.call(
79
+
backfill_state_subject,
80
+
waiting: 100,
81
+
sending: backfill_state.IsBackfilling,
82
+
)
83
+
84
+
let component = backfill_button.component(db, backfill_state_subject)
85
+
let assert Ok(runtime) =
86
+
lustre.start_server_component(component, #(is_admin, backfilling))
87
+
88
+
let self = process.new_subject()
89
+
let selector = process.new_selector() |> process.select(self)
90
+
91
+
server_component.register_subject(self)
92
+
|> lustre.send(to: runtime)
93
+
94
+
#(BackfillButtonSocket(component: runtime, self: self), option.Some(selector))
95
+
}
96
+
97
+
fn loop_backfill_button_socket(
98
+
state: BackfillButtonSocket,
99
+
message: mist.WebsocketMessage(BackfillButtonSocketMessage),
100
+
connection: mist.WebsocketConnection,
101
+
) -> mist.Next(BackfillButtonSocket, BackfillButtonSocketMessage) {
102
+
case message {
103
+
mist.Text(json_string) -> {
104
+
case json.parse(json_string, server_component.runtime_message_decoder()) {
105
+
Ok(runtime_message) -> lustre.send(state.component, runtime_message)
106
+
Error(_) -> Nil
107
+
}
108
+
109
+
mist.continue(state)
110
+
}
111
+
112
+
mist.Binary(_) -> mist.continue(state)
113
+
114
+
mist.Custom(client_message) -> {
115
+
let json_obj = server_component.client_message_to_json(client_message)
116
+
let assert Ok(_) =
117
+
mist.send_text_frame(connection, json.to_string(json_obj))
118
+
119
+
mist.continue(state)
120
+
}
121
+
122
+
mist.Closed | mist.Shutdown -> mist.stop()
123
+
}
124
+
}
125
+
126
+
fn close_backfill_button_socket(state: BackfillButtonSocket) -> Nil {
127
+
lustre.shutdown()
128
+
|> lustre.send(to: state.component)
129
+
}
+88
-80
server/src/pages/index.gleam
+88
-80
server/src/pages/index.gleam
···
1
+
import components/button
1
2
import components/collection_table
2
3
import components/layout
3
-
import components/stats_card
4
+
import components/sparkline
4
5
import database
5
-
import gleam/list
6
+
import format
6
7
import gleam/option.{type Option}
7
8
import lustre/attribute
8
9
import lustre/element.{type Element}
9
10
import lustre/element/html
11
+
import lustre/server_component
10
12
import sqlight
11
13
12
14
/// Page data aggregated from database queries
13
15
pub type IndexData {
14
16
IndexData(
17
+
record_count: Int,
15
18
lexicon_count: Int,
16
19
actor_count: Int,
17
20
collection_stats: List(database.CollectionStat),
18
21
record_lexicons: List(database.Lexicon),
22
+
record_activity: List(database.ActivityPoint),
19
23
)
20
24
}
21
25
···
31
35
32
36
/// Fetch all data needed for the index page
33
37
fn fetch_data(db: sqlight.Connection) -> IndexData {
38
+
let record_count = case database.get_record_count(db) {
39
+
Ok(count) -> count
40
+
Error(_) -> 0
41
+
}
42
+
34
43
let lexicon_count = case database.get_lexicon_count(db) {
35
44
Ok(count) -> count
36
45
Error(_) -> 0
···
51
60
Error(_) -> []
52
61
}
53
62
63
+
let record_activity = case database.get_record_activity(db, 168) {
64
+
Ok(activity) -> activity
65
+
Error(_) -> []
66
+
}
67
+
54
68
IndexData(
69
+
record_count: record_count,
55
70
lexicon_count: lexicon_count,
56
71
actor_count: actor_count,
57
72
collection_stats: collection_stats,
58
73
record_lexicons: record_lexicons,
74
+
record_activity: record_activity,
59
75
)
60
76
}
61
77
···
69
85
title: "ATProto Database Stats",
70
86
content: [
71
87
render_header(current_user, is_admin),
72
-
render_lexicons_section(data.lexicon_count),
73
-
render_actors_section(data.actor_count),
74
-
render_collections_section(data.collection_stats, data.record_lexicons),
88
+
render_stats_section(data.record_count, data.lexicon_count, data.actor_count),
89
+
render_activity_section(data.record_activity),
90
+
render_collections_section(
91
+
data.collection_stats,
92
+
data.record_lexicons,
93
+
is_admin,
94
+
),
75
95
],
76
96
)
77
97
}
···
79
99
/// Render the page header with title and action buttons
80
100
fn render_header(
81
101
current_user: Option(#(String, String)),
82
-
is_admin: Bool,
102
+
_is_admin: Bool,
83
103
) -> Element(msg) {
84
104
let action_buttons = case current_user {
85
105
option.Some(_) -> {
86
-
// Build list of action buttons based on permissions
87
-
let backfill_button = case is_admin {
88
-
True -> [
89
-
html.form(
90
-
[
91
-
attribute.method("post"),
92
-
attribute.action("/backfill"),
93
-
attribute.class("inline"),
94
-
],
95
-
[
96
-
html.button(
97
-
[
98
-
attribute.type_("submit"),
99
-
attribute.class(
100
-
"bg-blue-600 hover:bg-blue-700 text-white font-semibold py-2 px-4 rounded-lg transition-colors shadow-sm",
101
-
),
102
-
],
103
-
[element.text("Backfill Collections")],
104
-
),
105
-
],
106
-
),
107
-
]
108
-
False -> []
109
-
}
110
-
111
106
let common_buttons = [
112
-
html.a(
113
-
[
114
-
attribute.href("/graphiql"),
115
-
attribute.class(
116
-
"bg-purple-600 hover:bg-purple-700 text-white font-semibold py-2 px-4 rounded-lg transition-colors shadow-sm",
117
-
),
118
-
],
119
-
[element.text("Open GraphiQL")],
120
-
),
121
-
html.a(
122
-
[
123
-
attribute.href("/upload"),
124
-
attribute.class(
125
-
"bg-green-600 hover:bg-green-700 text-white font-semibold py-2 px-4 rounded-lg transition-colors shadow-sm",
126
-
),
127
-
],
128
-
[element.text("Upload Blob")],
129
-
),
107
+
button.link(href: "/graphiql", text: "Open GraphiQL"),
108
+
button.link(href: "/upload", text: "Upload Blob"),
130
109
]
131
110
132
111
[
133
-
html.div(
134
-
[attribute.class("flex gap-3")],
135
-
list.append(backfill_button, common_buttons),
136
-
),
112
+
html.div([attribute.class("flex gap-3")], common_buttons),
137
113
]
138
114
}
139
115
option.None -> []
···
142
118
html.div([attribute.class("mb-8")], [
143
119
// Title and user info row
144
120
html.div([attribute.class("flex justify-between items-center mb-4")], [
145
-
html.h1([attribute.class("text-4xl font-bold text-gray-900")], [
121
+
html.h1([attribute.class("text-4xl font-bold text-zinc-200")], [
146
122
element.text("quickslice"),
147
123
]),
148
124
render_user_section(current_user),
···
157
133
option.Some(#(_did, handle)) -> {
158
134
// User is logged in
159
135
html.div([attribute.class("flex items-center gap-3")], [
160
-
html.span([attribute.class("text-gray-700")], [
136
+
html.span([attribute.class("text-zinc-300")], [
161
137
element.text("Logged in as "),
162
-
html.span([attribute.class("font-semibold text-gray-900")], [
138
+
html.span([attribute.class("font-semibold text-zinc-200")], [
163
139
element.text("@" <> handle),
164
140
]),
165
141
]),
···
170
146
[
171
147
attribute.type_("submit"),
172
148
attribute.class(
173
-
"bg-gray-600 hover:bg-gray-700 text-white font-semibold py-2 px-4 rounded-lg transition-colors shadow-sm",
149
+
"px-4 py-2 text-sm text-zinc-400 border border-zinc-800 hover:border-zinc-700 hover:text-zinc-300 rounded transition-colors cursor-pointer",
174
150
),
175
151
],
176
152
[element.text("Logout")],
···
193
169
attribute.name("loginHint"),
194
170
attribute.placeholder("your-handle.bsky.social"),
195
171
attribute.class(
196
-
"px-3 py-2 border border-gray-300 rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500",
172
+
"px-3 py-2 bg-zinc-900 border border-zinc-800 rounded text-sm text-zinc-300 focus:outline-none focus:border-zinc-700",
197
173
),
198
174
attribute.attribute("required", ""),
199
175
]),
···
201
177
[
202
178
attribute.type_("submit"),
203
179
attribute.class(
204
-
"bg-blue-600 hover:bg-blue-700 text-white font-semibold py-2 px-4 rounded-lg transition-colors shadow-sm",
180
+
"px-4 py-2 text-sm text-zinc-300 bg-zinc-800 hover:bg-zinc-700 rounded transition-colors cursor-pointer",
205
181
),
206
182
],
207
183
[element.text("Login")],
···
212
188
}
213
189
}
214
190
215
-
/// Render the lexicons statistics section
216
-
fn render_lexicons_section(lexicon_count: Int) -> Element(msg) {
217
-
html.div([attribute.class("mb-8")], [
218
-
html.h2([attribute.class("text-2xl font-semibold text-gray-700 mb-4")], [
219
-
element.text("Lexicons"),
191
+
/// Render the combined statistics section
192
+
fn render_stats_section(record_count: Int, lexicon_count: Int, actor_count: Int) -> Element(msg) {
193
+
html.div([attribute.class("mb-8 grid grid-cols-3 gap-4")], [
194
+
// Total records stat card
195
+
html.div([attribute.class("bg-zinc-800/50 rounded p-4")], [
196
+
html.div([attribute.class("text-sm text-zinc-500 mb-1")], [
197
+
element.text("Total Records"),
198
+
]),
199
+
html.div([attribute.class("text-2xl font-semibold text-zinc-200")], [
200
+
element.text(format.format_number(record_count)),
201
+
]),
220
202
]),
221
-
stats_card.card(
222
-
count: lexicon_count,
223
-
description: "Lexicon schemas loaded",
224
-
color: "purple",
225
-
),
203
+
// Actors stat card
204
+
html.div([attribute.class("bg-zinc-800/50 rounded p-4")], [
205
+
html.div([attribute.class("text-sm text-zinc-500 mb-1")], [
206
+
element.text("Total Actors"),
207
+
]),
208
+
html.div([attribute.class("text-2xl font-semibold text-zinc-200")], [
209
+
element.text(format.format_number(actor_count)),
210
+
]),
211
+
]),
212
+
// Lexicons stat card
213
+
html.div([attribute.class("bg-zinc-800/50 rounded p-4")], [
214
+
html.div([attribute.class("text-sm text-zinc-500 mb-1")], [
215
+
element.text("Total Lexicons"),
216
+
]),
217
+
html.div([attribute.class("text-2xl font-semibold text-zinc-200")], [
218
+
element.text(format.format_number(lexicon_count)),
219
+
]),
220
+
]),
226
221
])
227
222
}
228
223
229
-
/// Render the actors statistics section
230
-
fn render_actors_section(actor_count: Int) -> Element(msg) {
224
+
/// Render the activity chart section
225
+
fn render_activity_section(
226
+
activity: List(database.ActivityPoint),
227
+
) -> Element(msg) {
231
228
html.div([attribute.class("mb-8")], [
232
-
html.h2([attribute.class("text-2xl font-semibold text-gray-700 mb-4")], [
233
-
element.text("Actors"),
229
+
html.div([attribute.class("bg-zinc-800/50 rounded p-4")], [
230
+
html.div([attribute.class("text-sm text-zinc-500 mb-3")], [
231
+
element.text("Activity (Last 7 Days)"),
232
+
]),
233
+
sparkline.view(activity),
234
234
]),
235
-
stats_card.card(
236
-
count: actor_count,
237
-
description: "Total actors indexed",
238
-
color: "blue",
239
-
),
240
235
])
241
236
}
242
237
···
244
239
fn render_collections_section(
245
240
collection_stats: List(database.CollectionStat),
246
241
record_lexicons: List(database.Lexicon),
242
+
is_admin: Bool,
247
243
) -> Element(msg) {
244
+
let backfill_button = case is_admin {
245
+
True ->
246
+
server_component.element(
247
+
[attribute.id("backfill-button"), server_component.route("/backfill-ws")],
248
+
[],
249
+
)
250
+
False -> element.none()
251
+
}
252
+
248
253
html.div([], [
249
-
html.h2([attribute.class("text-2xl font-semibold text-gray-700 mb-4")], [
250
-
element.text("Collections"),
254
+
html.div([attribute.class("flex justify-between items-center mb-4")], [
255
+
html.h2([attribute.class("text-2xl font-semibold text-zinc-300")], [
256
+
element.text("Collections"),
257
+
]),
258
+
backfill_button,
251
259
]),
252
260
collection_table.view(collection_stats, record_lexicons),
253
261
])
+65
-70
server/src/pages/upload.gleam
+65
-70
server/src/pages/upload.gleam
···
15
15
fn render_header(handle: String) -> Element(msg) {
16
16
html.div([attribute.class("mb-8 flex justify-between items-center")], [
17
17
html.div([], [
18
-
html.h1([attribute.class("text-4xl font-bold text-gray-900 mb-2")], [
18
+
html.h1([attribute.class("text-4xl font-bold text-zinc-200 mb-2")], [
19
19
element.text("Upload Blob"),
20
20
]),
21
-
html.p([attribute.class("text-gray-600")], [
21
+
html.p([attribute.class("text-zinc-400")], [
22
22
element.text("Test the uploadBlob mutation by uploading a file"),
23
23
]),
24
24
]),
25
25
html.div([attribute.class("text-right")], [
26
-
html.p([attribute.class("text-sm text-gray-600")], [
26
+
html.p([attribute.class("text-sm text-zinc-500")], [
27
27
element.text("Logged in as"),
28
28
]),
29
-
html.p([attribute.class("text-lg font-semibold text-gray-900")], [
29
+
html.p([attribute.class("text-lg font-semibold text-zinc-200")], [
30
30
element.text("@" <> handle),
31
31
]),
32
32
html.a(
33
33
[
34
34
attribute.href("/"),
35
-
attribute.class("text-sm text-purple-600 hover:text-purple-700"),
35
+
attribute.class(
36
+
"text-sm text-zinc-400 hover:text-zinc-300 transition-colors",
37
+
),
36
38
],
37
39
[element.text("← Back to Home")],
38
40
),
···
42
44
43
45
/// Render the upload form with JavaScript
44
46
fn render_upload_form(oauth_token: String) -> Element(msg) {
45
-
html.div(
46
-
[
47
-
attribute.class(
48
-
"bg-white rounded-lg shadow-sm border border-gray-200 p-6",
49
-
),
50
-
],
51
-
[
52
-
html.form([attribute.id("uploadForm"), attribute.class("space-y-6")], [
47
+
html.div([attribute.class("bg-zinc-800/50 rounded p-6")], [
48
+
html.form([attribute.id("uploadForm"), attribute.class("space-y-6")], [
49
+
html.input([
50
+
attribute.type_("hidden"),
51
+
attribute.id("token"),
52
+
attribute.value(oauth_token),
53
+
]),
54
+
html.div([], [
55
+
html.label(
56
+
[
57
+
attribute.for("file"),
58
+
attribute.class("block text-sm font-medium text-zinc-300 mb-2"),
59
+
],
60
+
[element.text("File to Upload")],
61
+
),
53
62
html.input([
54
-
attribute.type_("hidden"),
55
-
attribute.id("token"),
56
-
attribute.value(oauth_token),
57
-
]),
58
-
html.div([], [
59
-
html.label(
60
-
[
61
-
attribute.for("file"),
62
-
attribute.class("block text-sm font-medium text-gray-700 mb-2"),
63
-
],
64
-
[element.text("File to Upload")],
63
+
attribute.type_("file"),
64
+
attribute.id("file"),
65
+
attribute.name("file"),
66
+
attribute.class(
67
+
"w-full px-3 py-2 bg-zinc-900 border border-zinc-700 rounded text-zinc-300 focus:outline-none focus:border-zinc-600",
65
68
),
66
-
html.input([
67
-
attribute.type_("file"),
68
-
attribute.id("file"),
69
-
attribute.name("file"),
70
-
attribute.class(
71
-
"w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-purple-500 focus:border-purple-500",
72
-
),
73
-
attribute.attribute("required", ""),
74
-
]),
75
-
html.p([attribute.class("mt-1 text-sm text-gray-500")], [
76
-
element.text("Select an image or any file to upload as a blob"),
77
-
]),
69
+
attribute.attribute("required", ""),
78
70
]),
79
-
html.div([], [
80
-
html.button(
81
-
[
82
-
attribute.type_("submit"),
83
-
attribute.class(
84
-
"w-full bg-purple-600 hover:bg-purple-700 text-white font-semibold py-3 px-4 rounded-lg transition-colors shadow-sm",
85
-
),
86
-
],
87
-
[element.text("Upload Blob")],
88
-
),
71
+
html.p([attribute.class("mt-1 text-sm text-zinc-500")], [
72
+
element.text("Select an image or any file to upload as a blob"),
89
73
]),
90
74
]),
91
-
html.div([attribute.id("result"), attribute.class("mt-6 hidden")], [
92
-
html.h2([attribute.class("text-lg font-semibold text-gray-900 mb-2")], [
93
-
element.text("Result:"),
94
-
]),
95
-
html.div(
75
+
html.div([], [
76
+
html.button(
96
77
[
97
-
attribute.id("resultContent"),
78
+
attribute.type_("submit"),
98
79
attribute.class(
99
-
"bg-gray-50 rounded-lg p-4 border border-gray-200 font-mono text-sm overflow-auto",
80
+
"w-full bg-zinc-700 hover:bg-zinc-600 text-zinc-200 font-semibold py-3 px-4 rounded transition-colors",
100
81
),
101
82
],
102
-
[],
83
+
[element.text("Upload Blob")],
103
84
),
104
85
]),
105
-
html.div([attribute.id("error"), attribute.class("mt-6 hidden")], [
106
-
html.h2([attribute.class("text-lg font-semibold text-red-900 mb-2")], [
107
-
element.text("Error:"),
108
-
]),
109
-
html.div(
110
-
[
111
-
attribute.id("errorContent"),
112
-
attribute.class(
113
-
"bg-red-50 rounded-lg p-4 border border-red-200 text-red-700 font-mono text-sm overflow-auto",
114
-
),
115
-
],
116
-
[],
117
-
),
86
+
]),
87
+
html.div([attribute.id("result"), attribute.class("mt-6 hidden")], [
88
+
html.h2([attribute.class("text-lg font-semibold text-zinc-200 mb-2")], [
89
+
element.text("Result:"),
90
+
]),
91
+
html.div(
92
+
[
93
+
attribute.id("resultContent"),
94
+
attribute.class(
95
+
"bg-zinc-900 rounded p-4 border border-zinc-700 text-zinc-300 font-mono text-sm overflow-auto max-h-96",
96
+
),
97
+
],
98
+
[],
99
+
),
100
+
]),
101
+
html.div([attribute.id("error"), attribute.class("mt-6 hidden")], [
102
+
html.h2([attribute.class("text-lg font-semibold text-red-400 mb-2")], [
103
+
element.text("Error:"),
118
104
]),
119
-
render_upload_script(),
120
-
],
121
-
)
105
+
html.div(
106
+
[
107
+
attribute.id("errorContent"),
108
+
attribute.class(
109
+
"bg-red-950 rounded p-4 border border-red-900 text-red-300 font-mono text-sm overflow-auto max-h-96",
110
+
),
111
+
],
112
+
[],
113
+
),
114
+
]),
115
+
render_upload_script(),
116
+
])
122
117
}
123
118
124
119
/// Render the JavaScript for handling file upload
+54
-28
server/src/server.gleam
+54
-28
server/src/server.gleam
···
1
1
import argv
2
2
import backfill
3
+
import backfill_state
3
4
import database
4
5
import dotenv_gleam
5
6
import envoy
···
17
18
import jetstream_consumer
18
19
import logging
19
20
import lustre/element
21
+
import lustre_handlers
20
22
import mist
21
23
import oauth/handlers
22
24
import oauth/session
···
36
38
plc_url: String,
37
39
oauth_config: handlers.OAuthConfig,
38
40
admin_dids: List(String),
41
+
backfill_state: process.Subject(backfill_state.Message),
39
42
)
40
43
}
41
44
···
340
343
Error(_) -> "https://plc.directory"
341
344
}
342
345
346
+
// Start backfill state actor to track backfill status across requests
347
+
let assert Ok(backfill_state_subject) = backfill_state.start()
348
+
logging.log(logging.Info, "[server] Backfill state actor initialized")
349
+
343
350
let ctx =
344
351
Context(
345
352
db: db,
···
347
354
plc_url: plc_url,
348
355
oauth_config: oauth_config,
349
356
admin_dids: admin_dids,
357
+
backfill_state: backfill_state_subject,
350
358
)
351
359
352
360
let handler = fn(req) { handle_request(req, ctx) }
···
359
367
// Create Wisp handler converted to Mist format
360
368
let wisp_handler = wisp_mist.handler(handler, secret_key_base)
361
369
362
-
// Wrap it to intercept WebSocket upgrades
370
+
// Wrap it to intercept WebSocket upgrades and serve Lustre runtime
363
371
let mist_handler = fn(req: request.Request(mist.Connection)) {
364
-
// Check if this is a WebSocket upgrade request to /graphql
365
372
let upgrade_header = request.get_header(req, "upgrade")
366
373
let path = request.path_segments(req)
367
374
368
-
case upgrade_header, path {
369
-
Ok(upgrade_value), ["graphql"] | Ok(upgrade_value), ["", "graphql"] -> {
370
-
// Check if upgrade header contains "websocket" (case-insensitive)
371
-
case string.lowercase(upgrade_value) {
372
-
"websocket" -> {
373
-
logging.log(
374
-
logging.Info,
375
-
"[server] Handling WebSocket upgrade for /graphql",
376
-
)
377
-
// Handle WebSocket upgrade
378
-
graphql_ws_handler.handle_websocket(
379
-
req,
380
-
ctx.db,
381
-
ctx.auth_base_url,
382
-
ctx.plc_url,
383
-
)
375
+
case path {
376
+
// Serve Lustre client runtime
377
+
["lustre", "runtime.mjs"] -> lustre_handlers.serve_lustre_runtime()
378
+
379
+
// Backfill button WebSocket
380
+
["backfill-ws"] -> {
381
+
case upgrade_header {
382
+
Ok(upgrade_value) -> {
383
+
case string.lowercase(upgrade_value) {
384
+
"websocket" -> {
385
+
logging.log(logging.Info, "[server] WebSocket upgrade for /backfill-ws")
386
+
lustre_handlers.serve_backfill_button(
387
+
req,
388
+
ctx.db,
389
+
ctx.backfill_state,
390
+
)
391
+
}
392
+
_ -> wisp_handler(req)
393
+
}
384
394
}
385
-
_ -> {
386
-
logging.log(
387
-
logging.Warning,
388
-
"[server] Unknown upgrade type: " <> upgrade_value,
389
-
)
390
-
wisp_handler(req)
395
+
_ -> wisp_handler(req)
396
+
}
397
+
}
398
+
399
+
// GraphQL WebSocket
400
+
["graphql"] | ["", "graphql"] -> {
401
+
case upgrade_header {
402
+
Ok(upgrade_value) -> {
403
+
case string.lowercase(upgrade_value) {
404
+
"websocket" -> {
405
+
logging.log(
406
+
logging.Info,
407
+
"[server] Handling WebSocket upgrade for /graphql",
408
+
)
409
+
graphql_ws_handler.handle_websocket(
410
+
req,
411
+
ctx.db,
412
+
ctx.auth_base_url,
413
+
ctx.plc_url,
414
+
)
415
+
}
416
+
_ -> wisp_handler(req)
417
+
}
391
418
}
419
+
_ -> wisp_handler(req)
392
420
}
393
421
}
394
-
_, _ -> {
395
-
// Pass to Wisp handler for regular HTTP requests
396
-
wisp_handler(req)
397
-
}
422
+
423
+
_ -> wisp_handler(req)
398
424
}
399
425
}
400
426