search for standard sites
pub-search.waow.tech
search
zig
blog
atproto
batch dashboard queries into single HTTP request (5 → 1)
add batch query support to turso client for pipeline API,
reduce dashboard data fetching from 5 round trips to 1.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
+281
-88
+93
-81
backend/src/dashboard_data.zig
+93
-81
backend/src/dashboard_data.zig
···
2
2
const json = std.json;
3
3
const Allocator = std.mem.Allocator;
4
4
const db = @import("db/mod.zig");
5
-
const zql = @import("zql");
6
5
7
6
/// All data needed to render the dashboard
8
7
pub const DashboardData = struct {
···
16
15
top_pubs_json: []const u8,
17
16
};
18
17
19
-
pub fn fetch(alloc: Allocator) !DashboardData {
20
-
const stats = db.getStats();
21
-
const doc_types = getDocTypeStats();
18
+
// all dashboard queries batched into one request
19
+
const STATS_SQL =
20
+
\\SELECT
21
+
\\ (SELECT COUNT(*) FROM documents) as docs,
22
+
\\ (SELECT COUNT(*) FROM publications) as pubs,
23
+
\\ (SELECT total_searches FROM stats WHERE id = 1) as searches,
24
+
\\ (SELECT total_errors FROM stats WHERE id = 1) as errors,
25
+
\\ (SELECT service_started_at FROM stats WHERE id = 1) as started_at
26
+
;
22
27
23
-
return .{
24
-
.started_at = stats.started_at,
25
-
.searches = stats.searches,
26
-
.publications = stats.publications,
27
-
.articles = doc_types.articles,
28
-
.looseleafs = doc_types.looseleafs,
29
-
.tags_json = db.getTags(alloc) catch "[]",
30
-
.timeline_json = getDocsByDate(alloc) catch "[]",
31
-
.top_pubs_json = getTopPublications(alloc) catch "[]",
32
-
};
33
-
}
28
+
const DOC_TYPES_SQL =
29
+
\\SELECT
30
+
\\ SUM(CASE WHEN publication_uri != '' THEN 1 ELSE 0 END) as articles,
31
+
\\ SUM(CASE WHEN publication_uri = '' OR publication_uri IS NULL THEN 1 ELSE 0 END) as looseleafs
32
+
\\FROM documents
33
+
;
34
34
35
-
fn getDocTypeStats() struct { articles: i64, looseleafs: i64 } {
36
-
const client = db.getClient() orelse return .{ .articles = 0, .looseleafs = 0 };
35
+
const TAGS_SQL =
36
+
\\SELECT tag, COUNT(*) as count
37
+
\\FROM document_tags
38
+
\\GROUP BY tag
39
+
\\ORDER BY count DESC
40
+
\\LIMIT 100
41
+
;
37
42
38
-
var res = client.query(
39
-
\\SELECT
40
-
\\ SUM(CASE WHEN publication_uri != '' THEN 1 ELSE 0 END) as articles,
41
-
\\ SUM(CASE WHEN publication_uri = '' OR publication_uri IS NULL THEN 1 ELSE 0 END) as looseleafs
42
-
\\FROM documents
43
-
, &.{}) catch return .{ .articles = 0, .looseleafs = 0 };
44
-
defer res.deinit();
45
-
46
-
const row = res.first() orelse return .{ .articles = 0, .looseleafs = 0 };
47
-
return .{ .articles = row.int(0), .looseleafs = row.int(1) };
48
-
}
49
-
50
-
const DateCount = struct {
51
-
date: []const u8,
52
-
count: i64,
53
-
54
-
fn fromRow(row: db.Row) DateCount {
55
-
return .{ .date = row.text(0), .count = row.int(1) };
56
-
}
57
-
};
58
-
59
-
const DocsByDateQuery = zql.Query(
43
+
const TIMELINE_SQL =
60
44
\\SELECT DATE(created_at) as date, COUNT(*) as count
61
45
\\FROM documents
62
46
\\WHERE created_at IS NOT NULL AND created_at != ''
63
47
\\GROUP BY DATE(created_at)
64
48
\\ORDER BY date DESC
65
49
\\LIMIT 30
66
-
);
50
+
;
51
+
52
+
const TOP_PUBS_SQL =
53
+
\\SELECT p.name, p.base_path, COUNT(d.uri) as doc_count
54
+
\\FROM publications p
55
+
\\JOIN documents d ON d.publication_uri = p.uri
56
+
\\GROUP BY p.uri
57
+
\\ORDER BY doc_count DESC
58
+
\\LIMIT 8
59
+
;
67
60
68
-
fn getDocsByDate(alloc: Allocator) ![]const u8 {
61
+
pub fn fetch(alloc: Allocator) !DashboardData {
69
62
const client = db.getClient() orelse return error.NotInitialized;
70
63
71
-
var output: std.Io.Writer.Allocating = .init(alloc);
72
-
errdefer output.deinit();
64
+
// batch all 5 queries into one HTTP request
65
+
var batch = client.queryBatch(&.{
66
+
.{ .sql = STATS_SQL },
67
+
.{ .sql = DOC_TYPES_SQL },
68
+
.{ .sql = TAGS_SQL },
69
+
.{ .sql = TIMELINE_SQL },
70
+
.{ .sql = TOP_PUBS_SQL },
71
+
}) catch return error.QueryFailed;
72
+
defer batch.deinit();
73
73
74
-
var res = client.query(DocsByDateQuery.positional, &.{}) catch {
75
-
try output.writer.writeAll("[]");
76
-
return try output.toOwnedSlice();
74
+
// extract stats (query 0)
75
+
const stats_row = batch.getFirst(0);
76
+
const started_at = if (stats_row) |r| r.int(4) else 0;
77
+
const searches = if (stats_row) |r| r.int(2) else 0;
78
+
const publications = if (stats_row) |r| r.int(1) else 0;
79
+
80
+
// extract doc types (query 1)
81
+
const doc_row = batch.getFirst(1);
82
+
const articles = if (doc_row) |r| r.int(0) else 0;
83
+
const looseleafs = if (doc_row) |r| r.int(1) else 0;
84
+
85
+
return .{
86
+
.started_at = started_at,
87
+
.searches = searches,
88
+
.publications = publications,
89
+
.articles = articles,
90
+
.looseleafs = looseleafs,
91
+
.tags_json = try formatTagsJson(alloc, batch.get(2)),
92
+
.timeline_json = try formatTimelineJson(alloc, batch.get(3)),
93
+
.top_pubs_json = try formatPubsJson(alloc, batch.get(4)),
77
94
};
78
-
defer res.deinit();
95
+
}
96
+
97
+
fn formatTagsJson(alloc: Allocator, rows: []const db.Row) ![]const u8 {
98
+
var output: std.Io.Writer.Allocating = .init(alloc);
99
+
errdefer output.deinit();
79
100
80
101
var jw: json.Stringify = .{ .writer = &output.writer };
81
102
try jw.beginArray();
82
103
83
-
for (res.rows) |row| {
84
-
const dc = DateCount.fromRow(row);
104
+
for (rows) |row| {
85
105
try jw.beginObject();
86
-
try jw.objectField("date");
87
-
try jw.write(dc.date);
106
+
try jw.objectField("tag");
107
+
try jw.write(row.text(0));
88
108
try jw.objectField("count");
89
-
try jw.write(dc.count);
109
+
try jw.write(row.int(1));
90
110
try jw.endObject();
91
111
}
92
112
···
94
114
return try output.toOwnedSlice();
95
115
}
96
116
97
-
const TopPub = struct {
98
-
name: []const u8,
99
-
base_path: []const u8,
100
-
count: i64,
117
+
fn formatTimelineJson(alloc: Allocator, rows: []const db.Row) ![]const u8 {
118
+
var output: std.Io.Writer.Allocating = .init(alloc);
119
+
errdefer output.deinit();
120
+
121
+
var jw: json.Stringify = .{ .writer = &output.writer };
122
+
try jw.beginArray();
101
123
102
-
fn fromRow(row: db.Row) TopPub {
103
-
return .{ .name = row.text(0), .base_path = row.text(1), .count = row.int(2) };
124
+
for (rows) |row| {
125
+
try jw.beginObject();
126
+
try jw.objectField("date");
127
+
try jw.write(row.text(0));
128
+
try jw.objectField("count");
129
+
try jw.write(row.int(1));
130
+
try jw.endObject();
104
131
}
105
-
};
106
132
107
-
const TopPubsQuery = zql.Query(
108
-
\\SELECT p.name, p.base_path, COUNT(d.uri) as doc_count
109
-
\\FROM publications p
110
-
\\JOIN documents d ON d.publication_uri = p.uri
111
-
\\GROUP BY p.uri
112
-
\\ORDER BY doc_count DESC
113
-
\\LIMIT 8
114
-
);
133
+
try jw.endArray();
134
+
return try output.toOwnedSlice();
135
+
}
115
136
116
-
fn getTopPublications(alloc: Allocator) ![]const u8 {
117
-
const client = db.getClient() orelse return error.NotInitialized;
118
-
137
+
fn formatPubsJson(alloc: Allocator, rows: []const db.Row) ![]const u8 {
119
138
var output: std.Io.Writer.Allocating = .init(alloc);
120
139
errdefer output.deinit();
121
140
122
-
var res = client.query(TopPubsQuery.positional, &.{}) catch {
123
-
try output.writer.writeAll("[]");
124
-
return try output.toOwnedSlice();
125
-
};
126
-
defer res.deinit();
127
-
128
141
var jw: json.Stringify = .{ .writer = &output.writer };
129
142
try jw.beginArray();
130
143
131
-
for (res.rows) |row| {
132
-
const p = TopPub.fromRow(row);
144
+
for (rows) |row| {
133
145
try jw.beginObject();
134
146
try jw.objectField("name");
135
-
try jw.write(p.name);
147
+
try jw.write(row.text(0));
136
148
try jw.objectField("basePath");
137
-
try jw.write(p.base_path);
149
+
try jw.write(row.text(1));
138
150
try jw.objectField("count");
139
-
try jw.write(p.count);
151
+
try jw.write(row.int(2));
140
152
try jw.endObject();
141
153
}
142
154
+2
backend/src/db/mod.zig
+2
backend/src/db/mod.zig
···
10
10
pub const Client = turso.Client;
11
11
pub const Result = turso.Result;
12
12
pub const Row = turso.Row;
13
+
pub const BatchResult = turso.BatchResult;
14
+
pub const Statement = turso.Client.Statement;
13
15
14
16
var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{};
15
17
var client: ?turso.Client = null;
+81
-7
backend/src/db/result.zig
+81
-7
backend/src/db/result.zig
···
68
68
}
69
69
};
70
70
71
-
/// Navigate Turso's nested response format to get rows
72
-
fn getRowsFromParsed(value: json.Value) ?json.Array {
73
-
const results = value.object.get("results") orelse return null;
74
-
if (results != .array or results.array.items.len == 0) return null;
71
+
/// Batch result holding multiple query results
72
+
pub const BatchResult = struct {
73
+
allocator: Allocator,
74
+
parsed: ?json.Parsed(json.Value),
75
+
results: []const []const Row,
75
76
76
-
const first = results.array.items[0];
77
-
if (first != .object) return null;
77
+
pub fn parse(allocator: Allocator, response: []const u8, count: usize) !BatchResult {
78
+
const parsed = json.parseFromSlice(json.Value, allocator, response, .{}) catch {
79
+
return .{ .allocator = allocator, .parsed = null, .results = &.{} };
80
+
};
78
81
79
-
const resp = first.object.get("response") orelse return null;
82
+
const turso_results = parsed.value.object.get("results") orelse {
83
+
return .{ .allocator = allocator, .parsed = parsed, .results = &.{} };
84
+
};
85
+
86
+
if (turso_results != .array) {
87
+
return .{ .allocator = allocator, .parsed = parsed, .results = &.{} };
88
+
}
89
+
90
+
var all_results: std.ArrayList([]const Row) = .{};
91
+
errdefer {
92
+
for (all_results.items) |rows| allocator.free(rows);
93
+
all_results.deinit(allocator);
94
+
}
95
+
96
+
// turso returns: execute, close, execute, close, ...
97
+
// we want every other result (the executes, skip the closes)
98
+
var query_idx: usize = 0;
99
+
var i: usize = 0;
100
+
while (i < turso_results.array.items.len and query_idx < count) : (i += 2) {
101
+
const item = turso_results.array.items[i];
102
+
const json_rows = getRowsFromResult(item);
103
+
104
+
var rows: std.ArrayList(Row) = .{};
105
+
if (json_rows) |jr| {
106
+
for (jr.items) |row_item| {
107
+
if (row_item == .array) {
108
+
try rows.append(allocator, .{ .columns = row_item.array.items });
109
+
}
110
+
}
111
+
}
112
+
try all_results.append(allocator, try rows.toOwnedSlice(allocator));
113
+
query_idx += 1;
114
+
}
115
+
116
+
return .{
117
+
.allocator = allocator,
118
+
.parsed = parsed,
119
+
.results = try all_results.toOwnedSlice(allocator),
120
+
};
121
+
}
122
+
123
+
pub fn deinit(self: *BatchResult) void {
124
+
for (self.results) |rows| self.allocator.free(rows);
125
+
self.allocator.free(self.results);
126
+
if (self.parsed) |*p| p.deinit();
127
+
}
128
+
129
+
pub fn get(self: BatchResult, index: usize) []const Row {
130
+
if (index >= self.results.len) return &.{};
131
+
return self.results[index];
132
+
}
133
+
134
+
pub fn getFirst(self: BatchResult, index: usize) ?Row {
135
+
const rows = self.get(index);
136
+
if (rows.len == 0) return null;
137
+
return rows[0];
138
+
}
139
+
};
140
+
141
+
/// Get rows from a single result item in the results array
142
+
fn getRowsFromResult(item: json.Value) ?json.Array {
143
+
if (item != .object) return null;
144
+
145
+
const resp = item.object.get("response") orelse return null;
80
146
if (resp != .object) return null;
81
147
82
148
const res = resp.object.get("result") orelse return null;
···
86
152
if (rows != .array) return null;
87
153
88
154
return rows.array;
155
+
}
156
+
157
+
/// Navigate Turso's nested response format to get rows (first result only)
158
+
fn getRowsFromParsed(value: json.Value) ?json.Array {
159
+
const results = value.object.get("results") orelse return null;
160
+
if (results != .array or results.array.items.len == 0) return null;
161
+
162
+
return getRowsFromResult(results.array.items[0]);
89
163
}
90
164
91
165
/// Extract text from a Turso value (handles both raw and typed formats)
+105
backend/src/db/turso.zig
+105
backend/src/db/turso.zig
···
7
7
const result = @import("result.zig");
8
8
pub const Result = result.Result;
9
9
pub const Row = result.Row;
10
+
pub const BatchResult = result.BatchResult;
10
11
11
12
const URL_BUF_SIZE = 512;
12
13
const AUTH_BUF_SIZE = 512;
···
182
183
}
183
184
184
185
return try response_body.toOwnedSlice();
186
+
}
187
+
188
+
/// Statement for batch queries
189
+
pub const Statement = struct {
190
+
sql: []const u8,
191
+
args: []const []const u8 = &.{},
192
+
};
193
+
194
+
/// Execute multiple queries in a single HTTP request
195
+
pub fn queryBatch(self: *Client, statements: []const Statement) !BatchResult {
196
+
const response = try self.executeBatchRaw(statements);
197
+
defer self.allocator.free(response);
198
+
return BatchResult.parse(self.allocator, response, statements.len);
199
+
}
200
+
201
+
/// Execute batch and return raw JSON response
202
+
fn executeBatchRaw(self: *Client, statements: []const Statement) ![]const u8 {
203
+
self.mutex.lock();
204
+
defer self.mutex.unlock();
205
+
206
+
var url_buf: [URL_BUF_SIZE]u8 = undefined;
207
+
const url = std.fmt.bufPrint(&url_buf, "https://{s}/v2/pipeline", .{self.url}) catch
208
+
return error.UrlTooLong;
209
+
210
+
const body = try self.buildBatchRequestBody(statements);
211
+
defer self.allocator.free(body);
212
+
213
+
var auth_buf: [AUTH_BUF_SIZE]u8 = undefined;
214
+
const auth = std.fmt.bufPrint(&auth_buf, "Bearer {s}", .{self.token}) catch
215
+
return error.AuthTooLong;
216
+
217
+
var response_body: std.Io.Writer.Allocating = .init(self.allocator);
218
+
errdefer response_body.deinit();
219
+
220
+
const res = self.http_client.fetch(.{
221
+
.location = .{ .url = url },
222
+
.method = .POST,
223
+
.headers = .{
224
+
.content_type = .{ .override = "application/json" },
225
+
.authorization = .{ .override = auth },
226
+
},
227
+
.payload = body,
228
+
.response_writer = &response_body.writer,
229
+
}) catch |err| {
230
+
std.debug.print("turso batch request failed: {}\n", .{err});
231
+
return error.HttpError;
232
+
};
233
+
234
+
if (res.status != .ok) {
235
+
std.debug.print("turso batch error: {}\n", .{res.status});
236
+
return error.TursoError;
237
+
}
238
+
239
+
return try response_body.toOwnedSlice();
240
+
}
241
+
242
+
fn buildBatchRequestBody(self: *Client, statements: []const Statement) ![]const u8 {
243
+
var body: std.Io.Writer.Allocating = .init(self.allocator);
244
+
errdefer body.deinit();
245
+
246
+
var jw: json.Stringify = .{ .writer = &body.writer };
247
+
248
+
try jw.beginObject();
249
+
try jw.objectField("requests");
250
+
try jw.beginArray();
251
+
252
+
for (statements) |stmt| {
253
+
// execute statement
254
+
try jw.beginObject();
255
+
try jw.objectField("type");
256
+
try jw.write("execute");
257
+
try jw.objectField("stmt");
258
+
try jw.beginObject();
259
+
try jw.objectField("sql");
260
+
try jw.write(stmt.sql);
261
+
262
+
if (stmt.args.len > 0) {
263
+
try jw.objectField("args");
264
+
try jw.beginArray();
265
+
for (stmt.args) |arg| {
266
+
try jw.beginObject();
267
+
try jw.objectField("type");
268
+
try jw.write("text");
269
+
try jw.objectField("value");
270
+
try jw.write(arg);
271
+
try jw.endObject();
272
+
}
273
+
try jw.endArray();
274
+
}
275
+
276
+
try jw.endObject(); // stmt
277
+
try jw.endObject(); // execute request
278
+
279
+
// close after each statement
280
+
try jw.beginObject();
281
+
try jw.objectField("type");
282
+
try jw.write("close");
283
+
try jw.endObject();
284
+
}
285
+
286
+
try jw.endArray(); // requests
287
+
try jw.endObject(); // root
288
+
289
+
return try body.toOwnedSlice();
185
290
}
186
291
187
292
fn buildRequestBody(self: *Client, sql: []const u8, args: []const []const u8) ![]const u8 {