forked from
nekomimi.pet/wisp.place-monorepo
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
check manifest and calculate CIDs then compare if we need to reupload blobs
authored by
nekomimi.pet
and committed by
Tangled
e4f35dd7
390369a6
+576
-23
+18
-1
bun.lock
+18
-1
bun.lock
···
25
25
"elysia": "latest",
26
26
"iron-session": "^8.0.4",
27
27
"lucide-react": "^0.546.0",
28
+
"multiformats": "^13.4.1",
28
29
"react": "^19.2.0",
29
30
"react-dom": "^19.2.0",
30
31
"react-shiki": "^0.9.0",
···
641
642
642
643
"ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="],
643
644
644
-
"multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
645
+
"multiformats": ["multiformats@13.4.1", "", {}, "sha512-VqO6OSvLrFVAYYjgsr8tyv62/rCQhPgsZUXLTqoFLSgdkgiUYKYeArbt1uWLlEpkjxQe+P0+sHlbPEte1Bi06Q=="],
645
646
646
647
"negotiator": ["negotiator@0.6.3", "", {}, "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="],
647
648
···
857
858
858
859
"zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="],
859
860
861
+
"@atproto/api/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
862
+
863
+
"@atproto/common/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
864
+
865
+
"@atproto/common-web/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
866
+
867
+
"@atproto/jwk/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
868
+
869
+
"@atproto/lexicon/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
870
+
871
+
"@atproto/oauth-client/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
872
+
873
+
"@ipld/dag-cbor/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
874
+
860
875
"@radix-ui/react-collection/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="],
861
876
862
877
"@radix-ui/react-dialog/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="],
···
882
897
"send/encodeurl": ["encodeurl@1.0.2", "", {}, "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="],
883
898
884
899
"send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
900
+
901
+
"uint8arrays/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
885
902
886
903
"@tokenizer/inflate/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
887
904
+1
package.json
+1
package.json
+2
-2
public/editor/editor.tsx
+2
-2
public/editor/editor.tsx
···
748
748
749
749
<div className="p-4 bg-muted/30 rounded-lg border-l-4 border-yellow-500/50">
750
750
<div className="flex items-start gap-2">
751
-
<AlertCircle className="w-4 h-4 text-yellow-600 dark:text-yellow-400 mt-0.5 flex-shrink-0" />
751
+
<AlertCircle className="w-4 h-4 text-yellow-600 dark:text-yellow-400 mt-0.5 shrink-0" />
752
752
<div className="flex-1 space-y-1">
753
753
<p className="text-xs font-semibold text-yellow-600 dark:text-yellow-400">
754
754
Note about sites.wisp.place URLs
···
1120
1120
{skippedFiles.length > 0 && (
1121
1121
<div className="p-4 bg-yellow-500/10 border border-yellow-500/20 rounded-lg">
1122
1122
<div className="flex items-start gap-2 text-yellow-600 dark:text-yellow-400 mb-2">
1123
-
<AlertCircle className="w-4 h-4 mt-0.5 flex-shrink-0" />
1123
+
<AlertCircle className="w-4 h-4 mt-0.5 shrink-0" />
1124
1124
<div className="flex-1">
1125
1125
<span className="font-medium">
1126
1126
{skippedFiles.length} file{skippedFiles.length > 1 ? 's' : ''} skipped
-7
src/lib/db.ts
-7
src/lib/db.ts
···
244
244
245
245
const stateStore = {
246
246
async set(key: string, data: any) {
247
-
console.debug('[stateStore] set', key)
248
247
const expiresAt = Math.floor(Date.now() / 1000) + STATE_TIMEOUT;
249
248
await db`
250
249
INSERT INTO oauth_states (key, data, created_at, expires_at)
···
253
252
`;
254
253
},
255
254
async get(key: string) {
256
-
console.debug('[stateStore] get', key)
257
255
const now = Math.floor(Date.now() / 1000);
258
256
const result = await db`
259
257
SELECT data, expires_at
···
265
263
// Check if expired
266
264
const expiresAt = Number(result[0].expires_at);
267
265
if (expiresAt && now > expiresAt) {
268
-
console.debug('[stateStore] State expired, deleting', key);
269
266
await db`DELETE FROM oauth_states WHERE key = ${key}`;
270
267
return undefined;
271
268
}
···
273
270
return JSON.parse(result[0].data);
274
271
},
275
272
async del(key: string) {
276
-
console.debug('[stateStore] del', key)
277
273
await db`DELETE FROM oauth_states WHERE key = ${key}`;
278
274
}
279
275
};
280
276
281
277
const sessionStore = {
282
278
async set(sub: string, data: any) {
283
-
console.debug('[sessionStore] set', sub)
284
279
const expiresAt = Math.floor(Date.now() / 1000) + SESSION_TIMEOUT;
285
280
await db`
286
281
INSERT INTO oauth_sessions (sub, data, updated_at, expires_at)
···
292
287
`;
293
288
},
294
289
async get(sub: string) {
295
-
console.debug('[sessionStore] get', sub)
296
290
const now = Math.floor(Date.now() / 1000);
297
291
const result = await db`
298
292
SELECT data, expires_at
···
312
306
return JSON.parse(result[0].data);
313
307
},
314
308
async del(sub: string) {
315
-
console.debug('[sessionStore] del', sub)
316
309
await db`DELETE FROM oauth_sessions WHERE sub = ${sub}`;
317
310
}
318
311
};
-1
src/lib/oauth-client.ts
-1
src/lib/oauth-client.ts
+360
src/lib/wisp-utils.test.ts
+360
src/lib/wisp-utils.test.ts
···
5
5
processUploadedFiles,
6
6
createManifest,
7
7
updateFileBlobs,
8
+
computeCID,
9
+
extractBlobMap,
8
10
type UploadedFile,
9
11
type FileUploadResult,
10
12
} from './wisp-utils'
···
637
639
}
638
640
})
639
641
})
642
+
643
+
describe('computeCID', () => {
644
+
test('should compute CID for gzipped+base64 encoded content', () => {
645
+
// This simulates the actual flow: gzip -> base64 -> compute CID
646
+
const originalContent = Buffer.from('Hello, World!')
647
+
const gzipped = compressFile(originalContent)
648
+
const base64Content = Buffer.from(gzipped.toString('base64'), 'binary')
649
+
650
+
const cid = computeCID(base64Content)
651
+
652
+
// CID should be a valid CIDv1 string starting with 'bafkrei'
653
+
expect(cid).toMatch(/^bafkrei[a-z0-9]+$/)
654
+
expect(cid.length).toBeGreaterThan(10)
655
+
})
656
+
657
+
test('should compute deterministic CIDs for identical content', () => {
658
+
const content = Buffer.from('Test content for CID calculation')
659
+
const gzipped = compressFile(content)
660
+
const base64Content = Buffer.from(gzipped.toString('base64'), 'binary')
661
+
662
+
const cid1 = computeCID(base64Content)
663
+
const cid2 = computeCID(base64Content)
664
+
665
+
expect(cid1).toBe(cid2)
666
+
})
667
+
668
+
test('should compute different CIDs for different content', () => {
669
+
const content1 = Buffer.from('Content A')
670
+
const content2 = Buffer.from('Content B')
671
+
672
+
const gzipped1 = compressFile(content1)
673
+
const gzipped2 = compressFile(content2)
674
+
675
+
const base64Content1 = Buffer.from(gzipped1.toString('base64'), 'binary')
676
+
const base64Content2 = Buffer.from(gzipped2.toString('base64'), 'binary')
677
+
678
+
const cid1 = computeCID(base64Content1)
679
+
const cid2 = computeCID(base64Content2)
680
+
681
+
expect(cid1).not.toBe(cid2)
682
+
})
683
+
684
+
test('should handle empty content', () => {
685
+
const emptyContent = Buffer.from('')
686
+
const gzipped = compressFile(emptyContent)
687
+
const base64Content = Buffer.from(gzipped.toString('base64'), 'binary')
688
+
689
+
const cid = computeCID(base64Content)
690
+
691
+
expect(cid).toMatch(/^bafkrei[a-z0-9]+$/)
692
+
})
693
+
694
+
test('should compute same CID as PDS for base64-encoded content', () => {
695
+
// Test that binary encoding produces correct bytes for CID calculation
696
+
const testContent = Buffer.from('<!DOCTYPE html><html><body>Hello</body></html>')
697
+
const gzipped = compressFile(testContent)
698
+
const base64Content = Buffer.from(gzipped.toString('base64'), 'binary')
699
+
700
+
// Compute CID twice to ensure consistency
701
+
const cid1 = computeCID(base64Content)
702
+
const cid2 = computeCID(base64Content)
703
+
704
+
expect(cid1).toBe(cid2)
705
+
expect(cid1).toMatch(/^bafkrei/)
706
+
})
707
+
708
+
test('should use binary encoding for base64 strings', () => {
709
+
// This test verifies we're using the correct encoding method
710
+
// For base64 strings, 'binary' encoding ensures each character becomes exactly one byte
711
+
const content = Buffer.from('Test content')
712
+
const gzipped = compressFile(content)
713
+
const base64String = gzipped.toString('base64')
714
+
715
+
// Using binary encoding (what we use in production)
716
+
const base64Content = Buffer.from(base64String, 'binary')
717
+
718
+
// Verify the length matches the base64 string length
719
+
expect(base64Content.length).toBe(base64String.length)
720
+
721
+
// Verify CID is computed correctly
722
+
const cid = computeCID(base64Content)
723
+
expect(cid).toMatch(/^bafkrei/)
724
+
})
725
+
})
726
+
727
+
describe('extractBlobMap', () => {
728
+
test('should extract blob map from flat directory structure', () => {
729
+
const mockCid = CID.parse(TEST_CID_STRING)
730
+
const mockBlob = new BlobRef(mockCid, 'text/html', 100)
731
+
732
+
const directory: Directory = {
733
+
$type: 'place.wisp.fs#directory',
734
+
type: 'directory',
735
+
entries: [
736
+
{
737
+
name: 'index.html',
738
+
node: {
739
+
$type: 'place.wisp.fs#file',
740
+
type: 'file',
741
+
blob: mockBlob,
742
+
},
743
+
},
744
+
],
745
+
}
746
+
747
+
const blobMap = extractBlobMap(directory)
748
+
749
+
expect(blobMap.size).toBe(1)
750
+
expect(blobMap.has('index.html')).toBe(true)
751
+
752
+
const entry = blobMap.get('index.html')
753
+
expect(entry?.cid).toBe(TEST_CID_STRING)
754
+
expect(entry?.blobRef).toBe(mockBlob)
755
+
})
756
+
757
+
test('should extract blob map from nested directory structure', () => {
758
+
const mockCid1 = CID.parse(TEST_CID_STRING)
759
+
const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi')
760
+
761
+
const mockBlob1 = new BlobRef(mockCid1, 'text/html', 100)
762
+
const mockBlob2 = new BlobRef(mockCid2, 'text/css', 50)
763
+
764
+
const directory: Directory = {
765
+
$type: 'place.wisp.fs#directory',
766
+
type: 'directory',
767
+
entries: [
768
+
{
769
+
name: 'index.html',
770
+
node: {
771
+
$type: 'place.wisp.fs#file',
772
+
type: 'file',
773
+
blob: mockBlob1,
774
+
},
775
+
},
776
+
{
777
+
name: 'assets',
778
+
node: {
779
+
$type: 'place.wisp.fs#directory',
780
+
type: 'directory',
781
+
entries: [
782
+
{
783
+
name: 'styles.css',
784
+
node: {
785
+
$type: 'place.wisp.fs#file',
786
+
type: 'file',
787
+
blob: mockBlob2,
788
+
},
789
+
},
790
+
],
791
+
},
792
+
},
793
+
],
794
+
}
795
+
796
+
const blobMap = extractBlobMap(directory)
797
+
798
+
expect(blobMap.size).toBe(2)
799
+
expect(blobMap.has('index.html')).toBe(true)
800
+
expect(blobMap.has('assets/styles.css')).toBe(true)
801
+
802
+
expect(blobMap.get('index.html')?.cid).toBe(TEST_CID_STRING)
803
+
expect(blobMap.get('assets/styles.css')?.cid).toBe('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi')
804
+
})
805
+
806
+
test('should handle deeply nested directory structures', () => {
807
+
const mockCid = CID.parse(TEST_CID_STRING)
808
+
const mockBlob = new BlobRef(mockCid, 'text/javascript', 200)
809
+
810
+
const directory: Directory = {
811
+
$type: 'place.wisp.fs#directory',
812
+
type: 'directory',
813
+
entries: [
814
+
{
815
+
name: 'src',
816
+
node: {
817
+
$type: 'place.wisp.fs#directory',
818
+
type: 'directory',
819
+
entries: [
820
+
{
821
+
name: 'lib',
822
+
node: {
823
+
$type: 'place.wisp.fs#directory',
824
+
type: 'directory',
825
+
entries: [
826
+
{
827
+
name: 'utils.js',
828
+
node: {
829
+
$type: 'place.wisp.fs#file',
830
+
type: 'file',
831
+
blob: mockBlob,
832
+
},
833
+
},
834
+
],
835
+
},
836
+
},
837
+
],
838
+
},
839
+
},
840
+
],
841
+
}
842
+
843
+
const blobMap = extractBlobMap(directory)
844
+
845
+
expect(blobMap.size).toBe(1)
846
+
expect(blobMap.has('src/lib/utils.js')).toBe(true)
847
+
expect(blobMap.get('src/lib/utils.js')?.cid).toBe(TEST_CID_STRING)
848
+
})
849
+
850
+
test('should handle empty directory', () => {
851
+
const directory: Directory = {
852
+
$type: 'place.wisp.fs#directory',
853
+
type: 'directory',
854
+
entries: [],
855
+
}
856
+
857
+
const blobMap = extractBlobMap(directory)
858
+
859
+
expect(blobMap.size).toBe(0)
860
+
})
861
+
862
+
test('should correctly extract CID from BlobRef instances (not plain objects)', () => {
863
+
// This test verifies the fix: AT Protocol SDK returns BlobRef instances,
864
+
// not plain objects with $type and $link properties
865
+
const mockCid = CID.parse(TEST_CID_STRING)
866
+
const mockBlob = new BlobRef(mockCid, 'application/octet-stream', 500)
867
+
868
+
const directory: Directory = {
869
+
$type: 'place.wisp.fs#directory',
870
+
type: 'directory',
871
+
entries: [
872
+
{
873
+
name: 'test.bin',
874
+
node: {
875
+
$type: 'place.wisp.fs#file',
876
+
type: 'file',
877
+
blob: mockBlob,
878
+
},
879
+
},
880
+
],
881
+
}
882
+
883
+
const blobMap = extractBlobMap(directory)
884
+
885
+
// The fix: we call .toString() on the CID instance instead of accessing $link
886
+
expect(blobMap.get('test.bin')?.cid).toBe(TEST_CID_STRING)
887
+
expect(blobMap.get('test.bin')?.blobRef.ref.toString()).toBe(TEST_CID_STRING)
888
+
})
889
+
890
+
test('should handle multiple files in same directory', () => {
891
+
const mockCid1 = CID.parse(TEST_CID_STRING)
892
+
const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi')
893
+
const mockCid3 = CID.parse('bafkreieb3ixgchss44kw7xiavnkns47emdfsqbhcdfluo3p6n3o53fl3vq')
894
+
895
+
const mockBlob1 = new BlobRef(mockCid1, 'image/png', 1000)
896
+
const mockBlob2 = new BlobRef(mockCid2, 'image/png', 2000)
897
+
const mockBlob3 = new BlobRef(mockCid3, 'image/png', 3000)
898
+
899
+
const directory: Directory = {
900
+
$type: 'place.wisp.fs#directory',
901
+
type: 'directory',
902
+
entries: [
903
+
{
904
+
name: 'images',
905
+
node: {
906
+
$type: 'place.wisp.fs#directory',
907
+
type: 'directory',
908
+
entries: [
909
+
{
910
+
name: 'logo.png',
911
+
node: {
912
+
$type: 'place.wisp.fs#file',
913
+
type: 'file',
914
+
blob: mockBlob1,
915
+
},
916
+
},
917
+
{
918
+
name: 'banner.png',
919
+
node: {
920
+
$type: 'place.wisp.fs#file',
921
+
type: 'file',
922
+
blob: mockBlob2,
923
+
},
924
+
},
925
+
{
926
+
name: 'icon.png',
927
+
node: {
928
+
$type: 'place.wisp.fs#file',
929
+
type: 'file',
930
+
blob: mockBlob3,
931
+
},
932
+
},
933
+
],
934
+
},
935
+
},
936
+
],
937
+
}
938
+
939
+
const blobMap = extractBlobMap(directory)
940
+
941
+
expect(blobMap.size).toBe(3)
942
+
expect(blobMap.has('images/logo.png')).toBe(true)
943
+
expect(blobMap.has('images/banner.png')).toBe(true)
944
+
expect(blobMap.has('images/icon.png')).toBe(true)
945
+
})
946
+
947
+
test('should handle mixed directory and file structure', () => {
948
+
const mockCid1 = CID.parse(TEST_CID_STRING)
949
+
const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi')
950
+
const mockCid3 = CID.parse('bafkreieb3ixgchss44kw7xiavnkns47emdfsqbhcdfluo3p6n3o53fl3vq')
951
+
952
+
const directory: Directory = {
953
+
$type: 'place.wisp.fs#directory',
954
+
type: 'directory',
955
+
entries: [
956
+
{
957
+
name: 'index.html',
958
+
node: {
959
+
$type: 'place.wisp.fs#file',
960
+
type: 'file',
961
+
blob: new BlobRef(mockCid1, 'text/html', 100),
962
+
},
963
+
},
964
+
{
965
+
name: 'assets',
966
+
node: {
967
+
$type: 'place.wisp.fs#directory',
968
+
type: 'directory',
969
+
entries: [
970
+
{
971
+
name: 'styles.css',
972
+
node: {
973
+
$type: 'place.wisp.fs#file',
974
+
type: 'file',
975
+
blob: new BlobRef(mockCid2, 'text/css', 50),
976
+
},
977
+
},
978
+
],
979
+
},
980
+
},
981
+
{
982
+
name: 'README.md',
983
+
node: {
984
+
$type: 'place.wisp.fs#file',
985
+
type: 'file',
986
+
blob: new BlobRef(mockCid3, 'text/markdown', 200),
987
+
},
988
+
},
989
+
],
990
+
}
991
+
992
+
const blobMap = extractBlobMap(directory)
993
+
994
+
expect(blobMap.size).toBe(3)
995
+
expect(blobMap.has('index.html')).toBe(true)
996
+
expect(blobMap.has('assets/styles.css')).toBe(true)
997
+
expect(blobMap.has('README.md')).toBe(true)
998
+
})
999
+
})
+65
-2
src/lib/wisp-utils.ts
+65
-2
src/lib/wisp-utils.ts
···
2
2
import type { Record, Directory, File, Entry } from "../lexicons/types/place/wisp/fs";
3
3
import { validateRecord } from "../lexicons/types/place/wisp/fs";
4
4
import { gzipSync } from 'zlib';
5
+
import { CID } from 'multiformats/cid';
6
+
import { sha256 } from 'multiformats/hashes/sha2';
7
+
import * as raw from 'multiformats/codecs/raw';
8
+
import { createHash } from 'crypto';
9
+
import * as mf from 'multiformats';
5
10
6
11
export interface UploadedFile {
7
12
name: string;
···
48
53
}
49
54
50
55
/**
51
-
* Compress a file using gzip
56
+
* Compress a file using gzip with deterministic output
57
+
* Sets mtime to 0 to ensure identical content produces identical compressed output
52
58
*/
53
59
export function compressFile(content: Buffer): Buffer {
54
-
return gzipSync(content, { level: 9 });
60
+
return gzipSync(content, {
61
+
level: 9,
62
+
mtime: 0 // Fixed timestamp for deterministic compression
63
+
});
55
64
}
56
65
57
66
/**
···
65
74
const directoryMap = new Map<string, UploadedFile[]>();
66
75
67
76
for (const file of files) {
77
+
// Skip undefined/null files (defensive)
78
+
if (!file || !file.name) {
79
+
console.error('Skipping undefined or invalid file in processUploadedFiles');
80
+
continue;
81
+
}
82
+
68
83
// Remove any base folder name from the path
69
84
const normalizedPath = file.name.replace(/^[^\/]*\//, '');
70
85
const parts = normalizedPath.split('/');
···
239
254
240
255
return result;
241
256
}
257
+
258
+
/**
259
+
* Compute CID (Content Identifier) for blob content
260
+
* Uses the same algorithm as AT Protocol: CIDv1 with raw codec and SHA-256
261
+
* Based on @atproto/common/src/ipld.ts sha256RawToCid implementation
262
+
*/
263
+
export function computeCID(content: Buffer): string {
264
+
// Use node crypto to compute sha256 hash (same as AT Protocol)
265
+
const hash = createHash('sha256').update(content).digest();
266
+
// Create digest object from hash bytes
267
+
const digest = mf.digest.create(sha256.code, hash);
268
+
// Create CIDv1 with raw codec
269
+
const cid = CID.createV1(raw.code, digest);
270
+
return cid.toString();
271
+
}
272
+
273
+
/**
274
+
* Extract blob information from a directory tree
275
+
* Returns a map of file paths to their blob refs and CIDs
276
+
*/
277
+
export function extractBlobMap(
278
+
directory: Directory,
279
+
currentPath: string = ''
280
+
): Map<string, { blobRef: BlobRef; cid: string }> {
281
+
const blobMap = new Map<string, { blobRef: BlobRef; cid: string }>();
282
+
283
+
for (const entry of directory.entries) {
284
+
const fullPath = currentPath ? `${currentPath}/${entry.name}` : entry.name;
285
+
286
+
if ('type' in entry.node && entry.node.type === 'file') {
287
+
const fileNode = entry.node as File;
288
+
// AT Protocol SDK returns BlobRef class instances, not plain objects
289
+
// The ref is a CID instance that can be converted to string
290
+
if (fileNode.blob && fileNode.blob.ref) {
291
+
const cidString = fileNode.blob.ref.toString();
292
+
blobMap.set(fullPath, {
293
+
blobRef: fileNode.blob,
294
+
cid: cidString
295
+
});
296
+
}
297
+
} else if ('type' in entry.node && entry.node.type === 'directory') {
298
+
const subMap = extractBlobMap(entry.node as Directory, fullPath);
299
+
subMap.forEach((value, key) => blobMap.set(key, value));
300
+
}
301
+
}
302
+
303
+
return blobMap;
304
+
}
+130
-10
src/routes/wisp.ts
+130
-10
src/routes/wisp.ts
···
9
9
createManifest,
10
10
updateFileBlobs,
11
11
shouldCompressFile,
12
-
compressFile
12
+
compressFile,
13
+
computeCID,
14
+
extractBlobMap
13
15
} from '../lib/wisp-utils'
14
16
import { upsertSite } from '../lib/db'
15
17
import { logger } from '../lib/observability'
···
48
50
siteName: string;
49
51
files: File | File[]
50
52
};
53
+
54
+
console.log('=== UPLOAD FILES START ===');
55
+
console.log('Site name:', siteName);
56
+
console.log('Files received:', Array.isArray(files) ? files.length : 'single file');
51
57
52
58
try {
53
59
if (!siteName) {
···
106
112
107
113
// Create agent with OAuth session
108
114
const agent = new Agent((url, init) => auth.session.fetchHandler(url, init))
115
+
console.log('Agent created for DID:', auth.did);
116
+
117
+
// Try to fetch existing record to enable incremental updates
118
+
let existingBlobMap = new Map<string, { blobRef: any; cid: string }>();
119
+
console.log('Attempting to fetch existing record...');
120
+
try {
121
+
const rkey = siteName;
122
+
const existingRecord = await agent.com.atproto.repo.getRecord({
123
+
repo: auth.did,
124
+
collection: 'place.wisp.fs',
125
+
rkey: rkey
126
+
});
127
+
console.log('Existing record found!');
128
+
129
+
if (existingRecord.data.value && typeof existingRecord.data.value === 'object' && 'root' in existingRecord.data.value) {
130
+
const manifest = existingRecord.data.value as any;
131
+
existingBlobMap = extractBlobMap(manifest.root);
132
+
console.log(`Found existing manifest with ${existingBlobMap.size} files for incremental update`);
133
+
logger.info(`Found existing manifest with ${existingBlobMap.size} files for incremental update`);
134
+
}
135
+
} catch (error: any) {
136
+
console.log('No existing record found or error:', error?.message || error);
137
+
// Record doesn't exist yet, this is a new site
138
+
if (error?.status !== 400 && error?.error !== 'RecordNotFound') {
139
+
logger.warn('Failed to fetch existing record, proceeding with full upload', error);
140
+
}
141
+
}
109
142
110
143
// Convert File objects to UploadedFile format
111
144
// Elysia gives us File objects directly, handle both single file and array
···
113
146
const uploadedFiles: UploadedFile[] = [];
114
147
const skippedFiles: Array<{ name: string; reason: string }> = [];
115
148
116
-
149
+
console.log('Processing files, count:', fileArray.length);
117
150
118
151
for (let i = 0; i < fileArray.length; i++) {
119
152
const file = fileArray[i];
153
+
console.log(`Processing file ${i + 1}/${fileArray.length}:`, file.name, file.size, 'bytes');
120
154
121
155
// Skip files that are too large (limit to 100MB per file)
122
156
const maxSize = MAX_FILE_SIZE; // 100MB
···
135
169
// Compress and base64 encode ALL files
136
170
const compressedContent = compressFile(originalContent);
137
171
// Base64 encode the gzipped content to prevent PDS content sniffing
138
-
const base64Content = Buffer.from(compressedContent.toString('base64'), 'utf-8');
172
+
// Convert base64 string to bytes using binary encoding (each char becomes exactly one byte)
173
+
// This is what PDS receives and computes CID on
174
+
const base64Content = Buffer.from(compressedContent.toString('base64'), 'binary');
139
175
const compressionRatio = (compressedContent.length / originalContent.length * 100).toFixed(1);
176
+
console.log(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`);
140
177
logger.info(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`);
141
178
142
179
uploadedFiles.push({
143
180
name: file.name,
144
-
content: base64Content,
181
+
content: base64Content, // This is the gzipped+base64 content that will be uploaded and CID-computed
145
182
mimeType: originalMimeType,
146
183
size: base64Content.length,
147
184
compressed: true,
···
206
243
}
207
244
208
245
// Process files into directory structure
209
-
const { directory, fileCount } = processUploadedFiles(uploadedFiles);
246
+
console.log('Processing uploaded files into directory structure...');
247
+
console.log('uploadedFiles array length:', uploadedFiles.length);
248
+
console.log('uploadedFiles contents:', uploadedFiles.map((f, i) => `${i}: ${f?.name || 'UNDEFINED'}`));
210
249
211
-
// Upload files as blobs in parallel
250
+
// Filter out any undefined/null/invalid entries (defensive)
251
+
const validUploadedFiles = uploadedFiles.filter((f, i) => {
252
+
if (!f) {
253
+
console.error(`Filtering out undefined/null file at index ${i}`);
254
+
return false;
255
+
}
256
+
if (!f.name) {
257
+
console.error(`Filtering out file with no name at index ${i}:`, f);
258
+
return false;
259
+
}
260
+
if (!f.content) {
261
+
console.error(`Filtering out file with no content at index ${i}:`, f.name);
262
+
return false;
263
+
}
264
+
return true;
265
+
});
266
+
if (validUploadedFiles.length !== uploadedFiles.length) {
267
+
console.warn(`Filtered out ${uploadedFiles.length - validUploadedFiles.length} invalid files`);
268
+
}
269
+
console.log('validUploadedFiles length:', validUploadedFiles.length);
270
+
271
+
const { directory, fileCount } = processUploadedFiles(validUploadedFiles);
272
+
console.log('Directory structure created, file count:', fileCount);
273
+
274
+
// Upload files as blobs in parallel (or reuse existing blobs with matching CIDs)
275
+
console.log('Starting blob upload/reuse phase...');
212
276
// For compressed files, we upload as octet-stream and store the original MIME type in metadata
213
277
// For text/html files, we also use octet-stream as a workaround for PDS image pipeline issues
214
-
const uploadPromises = uploadedFiles.map(async (file, i) => {
278
+
const uploadPromises = validUploadedFiles.map(async (file, i) => {
215
279
try {
280
+
// Skip undefined files (shouldn't happen after filter, but defensive)
281
+
if (!file || !file.name) {
282
+
console.error(`ERROR: Undefined file at index ${i} in validUploadedFiles!`);
283
+
throw new Error(`Undefined file at index ${i}`);
284
+
}
285
+
286
+
// Compute CID for this file to check if it already exists
287
+
// Note: file.content is already gzipped+base64 encoded
288
+
const fileCID = computeCID(file.content);
289
+
290
+
// Normalize the file path for comparison (remove base folder prefix like "cobblemon/")
291
+
const normalizedPath = file.name.replace(/^[^\/]*\//, '');
292
+
293
+
// Check if we have an existing blob with the same CID
294
+
// Try both the normalized path and the full path
295
+
const existingBlob = existingBlobMap.get(normalizedPath) || existingBlobMap.get(file.name);
296
+
297
+
if (existingBlob && existingBlob.cid === fileCID) {
298
+
// Reuse existing blob - no need to upload
299
+
logger.info(`[File Upload] Reusing existing blob for: ${file.name} (CID: ${fileCID})`);
300
+
301
+
return {
302
+
result: {
303
+
hash: existingBlob.cid,
304
+
blobRef: existingBlob.blobRef,
305
+
...(file.compressed && {
306
+
encoding: 'gzip' as const,
307
+
mimeType: file.originalMimeType || file.mimeType,
308
+
base64: true
309
+
})
310
+
},
311
+
filePath: file.name,
312
+
sentMimeType: file.mimeType,
313
+
returnedMimeType: existingBlob.blobRef.mimeType,
314
+
reused: true
315
+
};
316
+
}
317
+
318
+
// File is new or changed - upload it
216
319
// If compressed, always upload as octet-stream
217
320
// Otherwise, workaround: PDS incorrectly processes text/html through image pipeline
218
321
const uploadMimeType = file.compressed || file.mimeType.startsWith('text/html')
···
220
323
: file.mimeType;
221
324
222
325
const compressionInfo = file.compressed ? ' (gzipped)' : '';
223
-
logger.info(`[File Upload] Uploading file: ${file.name} (original: ${file.mimeType}, sending as: ${uploadMimeType}, ${file.size} bytes${compressionInfo})`);
326
+
logger.info(`[File Upload] Uploading new/changed file: ${file.name} (original: ${file.mimeType}, sending as: ${uploadMimeType}, ${file.size} bytes${compressionInfo}, CID: ${fileCID})`);
224
327
225
328
const uploadResult = await agent.com.atproto.repo.uploadBlob(
226
329
file.content,
···
244
347
},
245
348
filePath: file.name,
246
349
sentMimeType: file.mimeType,
247
-
returnedMimeType: returnedBlobRef.mimeType
350
+
returnedMimeType: returnedBlobRef.mimeType,
351
+
reused: false
248
352
};
249
353
} catch (uploadError) {
250
354
logger.error('Upload failed for file', uploadError);
···
255
359
// Wait for all uploads to complete
256
360
const uploadedBlobs = await Promise.all(uploadPromises);
257
361
362
+
// Count reused vs uploaded blobs
363
+
const reusedCount = uploadedBlobs.filter(b => (b as any).reused).length;
364
+
const uploadedCount = uploadedBlobs.filter(b => !(b as any).reused).length;
365
+
console.log(`Blob statistics: ${reusedCount} reused, ${uploadedCount} uploaded, ${uploadedBlobs.length} total`);
366
+
logger.info(`Blob statistics: ${reusedCount} reused, ${uploadedCount} uploaded, ${uploadedBlobs.length} total`);
367
+
258
368
// Extract results and file paths in correct order
259
369
const uploadResults: FileUploadResult[] = uploadedBlobs.map(blob => blob.result);
260
370
const filePaths: string[] = uploadedBlobs.map(blob => blob.filePath);
261
371
262
372
// Update directory with file blobs
373
+
console.log('Updating directory with blob references...');
263
374
const updatedDirectory = updateFileBlobs(directory, uploadResults, filePaths);
264
375
265
376
// Create manifest
377
+
console.log('Creating manifest...');
266
378
const manifest = createManifest(siteName, updatedDirectory, fileCount);
379
+
console.log('Manifest created successfully');
267
380
268
381
// Use site name as rkey
269
382
const rkey = siteName;
270
383
271
384
let record;
272
385
try {
386
+
console.log('Putting record to PDS with rkey:', rkey);
273
387
record = await agent.com.atproto.repo.putRecord({
274
388
repo: auth.did,
275
389
collection: 'place.wisp.fs',
276
390
rkey: rkey,
277
391
record: manifest
278
392
});
393
+
console.log('Record successfully created on PDS:', record.data.uri);
279
394
} catch (putRecordError: any) {
395
+
console.error('FAILED to create record on PDS:', putRecordError);
280
396
logger.error('Failed to create record on PDS', putRecordError);
281
397
282
398
throw putRecordError;
···
292
408
fileCount,
293
409
siteName,
294
410
skippedFiles,
295
-
uploadedCount: uploadedFiles.length
411
+
uploadedCount: validUploadedFiles.length
296
412
};
297
413
414
+
console.log('=== UPLOAD FILES COMPLETE ===');
298
415
return result;
299
416
} catch (error) {
417
+
console.error('=== UPLOAD ERROR ===');
418
+
console.error('Error details:', error);
419
+
console.error('Stack trace:', error instanceof Error ? error.stack : 'N/A');
300
420
logger.error('Upload error', error, {
301
421
message: error instanceof Error ? error.message : 'Unknown error',
302
422
name: error instanceof Error ? error.name : undefined