forked from
microcosm.blue/Allegedly
Server tools to backfill, tail, mirror, and verify PLC logs
fjall: store serde_json::Value in db instead of storing RawValue
+37
-26
+20
-19
src/mirror/fjall.rs
+20
-19
src/mirror/fjall.rs
···
167
167
168
168
match sub_path {
169
169
"" => {
170
-
let parsed: Vec<serde_json::Value> = ops
171
-
.iter()
172
-
.filter(|op| !op.nullified)
173
-
.filter_map(|op| serde_json::from_str(op.operation.get()).ok())
174
-
.collect();
175
-
let data = doc::apply_op_log(did_str, &parsed);
170
+
let data = doc::apply_op_log(
171
+
did_str,
172
+
ops.iter()
173
+
.filter(|op| !op.nullified)
174
+
.map(|op| &op.operation),
175
+
);
176
176
let Some(data) = data else {
177
177
return Err(Error::from_string(
178
178
format!("DID not available: {did_str}"),
···
185
185
.body(serde_json::to_string(&doc).unwrap()))
186
186
}
187
187
"/log" => {
188
-
let log: Vec<serde_json::Value> = ops
188
+
let log: Vec<&serde_json::Value> = ops
189
189
.iter()
190
190
.filter(|op| !op.nullified)
191
-
.filter_map(|op| serde_json::from_str(op.operation.get()).ok())
191
+
.map(|op| &op.operation)
192
192
.collect();
193
193
Ok(Response::builder()
194
194
.content_type("application/json")
···
200
200
.map(|op| {
201
201
serde_json::json!({
202
202
"did": op.did,
203
-
"operation": serde_json::from_str::<serde_json::Value>(op.operation.get()).unwrap_or_default(),
203
+
"operation": op.operation,
204
204
"cid": op.cid,
205
205
"nullified": op.nullified,
206
206
"createdAt": op.created_at.to_rfc3339(),
···
212
212
.body(serde_json::to_string(&audit).unwrap()))
213
213
}
214
214
"/log/last" => {
215
-
let last =
216
-
ops.iter().filter(|op| !op.nullified).last().and_then(|op| {
217
-
serde_json::from_str::<serde_json::Value>(op.operation.get()).ok()
218
-
});
215
+
let last = ops
216
+
.iter()
217
+
.filter(|op| !op.nullified)
218
+
.last()
219
+
.map(|op| &op.operation);
219
220
let Some(last) = last else {
220
221
return Err(Error::from_string(
221
222
format!("DID not available: {did_str}"),
···
227
228
.body(serde_json::to_string(&last).unwrap()))
228
229
}
229
230
"/data" => {
230
-
let parsed: Vec<serde_json::Value> = ops
231
-
.iter()
232
-
.filter(|op| !op.nullified)
233
-
.filter_map(|op| serde_json::from_str(op.operation.get()).ok())
234
-
.collect();
235
-
let data = doc::apply_op_log(did_str, &parsed);
231
+
let data = doc::apply_op_log(
232
+
did_str,
233
+
ops.iter()
234
+
.filter(|op| !op.nullified)
235
+
.map(|op| &op.operation),
236
+
);
236
237
let Some(data) = data else {
237
238
return Err(Error::from_string(
238
239
format!("DID not available: {did_str}"),
+17
-7
src/plc_fjall.rs
+17
-7
src/plc_fjall.rs
···
1
-
use crate::{Dt, ExportPage, Op, PageBoundaryState};
1
+
use crate::{Dt, ExportPage, Op as CommonOp, PageBoundaryState};
2
2
use data_encoding::BASE32_NOPAD;
3
3
use fjall::{Database, Keyspace, KeyspaceCreateOptions, OwnedWriteBatch, PersistMode};
4
4
use serde::{Deserialize, Serialize};
···
76
76
);
77
77
Dt::from_timestamp_micros(micros as i64)
78
78
.ok_or_else(|| anyhow::anyhow!("invalid timestamp {micros}"))
79
+
}
80
+
81
+
// we have our own Op struct for fjall since we dont want to have to convert Value back to RawValue
82
+
#[derive(Debug, Serialize)]
83
+
pub struct Op {
84
+
pub did: String,
85
+
pub cid: String,
86
+
pub created_at: Dt,
87
+
pub nullified: bool,
88
+
pub operation: serde_json::Value,
79
89
}
80
90
81
91
// this is basically Op, but without the cid and created_at fields
···
86
96
#[serde(with = "serde_bytes")]
87
97
pub did: Vec<u8>,
88
98
pub nullified: bool,
89
-
pub operation: Box<serde_json::value::RawValue>,
99
+
pub operation: serde_json::Value,
90
100
}
91
101
92
102
#[derive(Clone)]
···
142
152
.map(Some)
143
153
}
144
154
145
-
pub fn insert_op(&self, batch: &mut OwnedWriteBatch, op: &Op) -> anyhow::Result<usize> {
155
+
pub fn insert_op(&self, batch: &mut OwnedWriteBatch, op: &CommonOp) -> anyhow::Result<usize> {
146
156
let pk = by_did_key(&op.did, &op.created_at, &op.cid)?;
147
157
if self.inner.by_did.get(&pk)?.is_some() {
148
158
return Ok(0);
···
155
165
let db_op = DbOp {
156
166
did: encoded_did,
157
167
nullified: op.nullified,
158
-
operation: op.operation.clone(),
168
+
operation: serde_json::to_value(&op.operation)?,
159
169
};
160
-
let value = rmp_serde::to_vec_named(&db_op)?;
170
+
let value = rmp_serde::to_vec(&db_op)?;
161
171
batch.insert(&self.inner.ops, &ts_key, &value);
162
172
batch.insert(&self.inner.by_did, &pk, &[]);
163
173
Ok(1)
···
180
190
181
191
let ts_bytes = key_rest
182
192
.get(..8)
183
-
.ok_or_else(|| anyhow::anyhow!("invalid length"))?;
193
+
.ok_or_else(|| anyhow::anyhow!("invalid length: {key_rest:?}"))?;
184
194
let cid_bytes = key_rest
185
195
.get(9..)
186
-
.ok_or_else(|| anyhow::anyhow!("invalid length"))?;
196
+
.ok_or_else(|| anyhow::anyhow!("invalid length: {key_rest:?}"))?;
187
197
188
198
let op_key = [ts_bytes, &[SEP][..], cid_bytes].concat();
189
199
let ts = decode_timestamp(ts_bytes)?;