forked from
nekomimi.pet/wisp.place-monorepo
Monorepo for wisp.place. A static site hosting service built on top of the AT Protocol.
check manifest and calculate CIDs then compare if we need to reupload blobs
+576
-23
+18
-1
bun.lock
+18
-1
bun.lock
···
25
25
"elysia": "latest",
26
26
"iron-session": "^8.0.4",
27
27
"lucide-react": "^0.546.0",
28
+
"multiformats": "^13.4.1",
28
29
"react": "^19.2.0",
29
30
"react-dom": "^19.2.0",
30
31
"react-shiki": "^0.9.0",
···
641
642
642
643
"ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="],
643
644
644
-
"multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
645
+
"multiformats": ["multiformats@13.4.1", "", {}, "sha512-VqO6OSvLrFVAYYjgsr8tyv62/rCQhPgsZUXLTqoFLSgdkgiUYKYeArbt1uWLlEpkjxQe+P0+sHlbPEte1Bi06Q=="],
645
646
646
647
"negotiator": ["negotiator@0.6.3", "", {}, "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="],
647
648
···
857
858
858
859
"zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="],
859
860
861
+
"@atproto/api/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
862
+
863
+
"@atproto/common/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
864
+
865
+
"@atproto/common-web/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
866
+
867
+
"@atproto/jwk/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
868
+
869
+
"@atproto/lexicon/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
870
+
871
+
"@atproto/oauth-client/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
872
+
873
+
"@ipld/dag-cbor/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
874
+
860
875
"@radix-ui/react-collection/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="],
861
876
862
877
"@radix-ui/react-dialog/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="],
···
882
897
"send/encodeurl": ["encodeurl@1.0.2", "", {}, "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w=="],
883
898
884
899
"send/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
900
+
901
+
"uint8arrays/multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
885
902
886
903
"@tokenizer/inflate/debug/ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
887
904
+1
package.json
+1
package.json
+2
-2
public/editor/editor.tsx
+2
-2
public/editor/editor.tsx
···
748
748
749
749
<div className="p-4 bg-muted/30 rounded-lg border-l-4 border-yellow-500/50">
750
750
<div className="flex items-start gap-2">
751
-
<AlertCircle className="w-4 h-4 text-yellow-600 dark:text-yellow-400 mt-0.5 flex-shrink-0" />
751
+
<AlertCircle className="w-4 h-4 text-yellow-600 dark:text-yellow-400 mt-0.5 shrink-0" />
752
752
<div className="flex-1 space-y-1">
753
753
<p className="text-xs font-semibold text-yellow-600 dark:text-yellow-400">
754
754
Note about sites.wisp.place URLs
···
1120
1120
{skippedFiles.length > 0 && (
1121
1121
<div className="p-4 bg-yellow-500/10 border border-yellow-500/20 rounded-lg">
1122
1122
<div className="flex items-start gap-2 text-yellow-600 dark:text-yellow-400 mb-2">
1123
-
<AlertCircle className="w-4 h-4 mt-0.5 flex-shrink-0" />
1123
+
<AlertCircle className="w-4 h-4 mt-0.5 shrink-0" />
1124
1124
<div className="flex-1">
1125
1125
<span className="font-medium">
1126
1126
{skippedFiles.length} file{skippedFiles.length > 1 ? 's' : ''} skipped
-7
src/lib/db.ts
-7
src/lib/db.ts
···
244
244
245
245
const stateStore = {
246
246
async set(key: string, data: any) {
247
-
console.debug('[stateStore] set', key)
248
247
const expiresAt = Math.floor(Date.now() / 1000) + STATE_TIMEOUT;
249
248
await db`
250
249
INSERT INTO oauth_states (key, data, created_at, expires_at)
···
253
252
`;
254
253
},
255
254
async get(key: string) {
256
-
console.debug('[stateStore] get', key)
257
255
const now = Math.floor(Date.now() / 1000);
258
256
const result = await db`
259
257
SELECT data, expires_at
···
265
263
// Check if expired
266
264
const expiresAt = Number(result[0].expires_at);
267
265
if (expiresAt && now > expiresAt) {
268
-
console.debug('[stateStore] State expired, deleting', key);
269
266
await db`DELETE FROM oauth_states WHERE key = ${key}`;
270
267
return undefined;
271
268
}
···
273
270
return JSON.parse(result[0].data);
274
271
},
275
272
async del(key: string) {
276
-
console.debug('[stateStore] del', key)
277
273
await db`DELETE FROM oauth_states WHERE key = ${key}`;
278
274
}
279
275
};
280
276
281
277
const sessionStore = {
282
278
async set(sub: string, data: any) {
283
-
console.debug('[sessionStore] set', sub)
284
279
const expiresAt = Math.floor(Date.now() / 1000) + SESSION_TIMEOUT;
285
280
await db`
286
281
INSERT INTO oauth_sessions (sub, data, updated_at, expires_at)
···
292
287
`;
293
288
},
294
289
async get(sub: string) {
295
-
console.debug('[sessionStore] get', sub)
296
290
const now = Math.floor(Date.now() / 1000);
297
291
const result = await db`
298
292
SELECT data, expires_at
···
312
306
return JSON.parse(result[0].data);
313
307
},
314
308
async del(sub: string) {
315
-
console.debug('[sessionStore] del', sub)
316
309
await db`DELETE FROM oauth_sessions WHERE sub = ${sub}`;
317
310
}
318
311
};
-1
src/lib/oauth-client.ts
-1
src/lib/oauth-client.ts
+360
src/lib/wisp-utils.test.ts
+360
src/lib/wisp-utils.test.ts
···
5
5
processUploadedFiles,
6
6
createManifest,
7
7
updateFileBlobs,
8
+
computeCID,
9
+
extractBlobMap,
8
10
type UploadedFile,
9
11
type FileUploadResult,
10
12
} from './wisp-utils'
···
637
639
}
638
640
})
639
641
})
642
+
643
+
describe('computeCID', () => {
644
+
test('should compute CID for gzipped+base64 encoded content', () => {
645
+
// This simulates the actual flow: gzip -> base64 -> compute CID
646
+
const originalContent = Buffer.from('Hello, World!')
647
+
const gzipped = compressFile(originalContent)
648
+
const base64Content = Buffer.from(gzipped.toString('base64'), 'binary')
649
+
650
+
const cid = computeCID(base64Content)
651
+
652
+
// CID should be a valid CIDv1 string starting with 'bafkrei'
653
+
expect(cid).toMatch(/^bafkrei[a-z0-9]+$/)
654
+
expect(cid.length).toBeGreaterThan(10)
655
+
})
656
+
657
+
test('should compute deterministic CIDs for identical content', () => {
658
+
const content = Buffer.from('Test content for CID calculation')
659
+
const gzipped = compressFile(content)
660
+
const base64Content = Buffer.from(gzipped.toString('base64'), 'binary')
661
+
662
+
const cid1 = computeCID(base64Content)
663
+
const cid2 = computeCID(base64Content)
664
+
665
+
expect(cid1).toBe(cid2)
666
+
})
667
+
668
+
test('should compute different CIDs for different content', () => {
669
+
const content1 = Buffer.from('Content A')
670
+
const content2 = Buffer.from('Content B')
671
+
672
+
const gzipped1 = compressFile(content1)
673
+
const gzipped2 = compressFile(content2)
674
+
675
+
const base64Content1 = Buffer.from(gzipped1.toString('base64'), 'binary')
676
+
const base64Content2 = Buffer.from(gzipped2.toString('base64'), 'binary')
677
+
678
+
const cid1 = computeCID(base64Content1)
679
+
const cid2 = computeCID(base64Content2)
680
+
681
+
expect(cid1).not.toBe(cid2)
682
+
})
683
+
684
+
test('should handle empty content', () => {
685
+
const emptyContent = Buffer.from('')
686
+
const gzipped = compressFile(emptyContent)
687
+
const base64Content = Buffer.from(gzipped.toString('base64'), 'binary')
688
+
689
+
const cid = computeCID(base64Content)
690
+
691
+
expect(cid).toMatch(/^bafkrei[a-z0-9]+$/)
692
+
})
693
+
694
+
test('should compute same CID as PDS for base64-encoded content', () => {
695
+
// Test that binary encoding produces correct bytes for CID calculation
696
+
const testContent = Buffer.from('<!DOCTYPE html><html><body>Hello</body></html>')
697
+
const gzipped = compressFile(testContent)
698
+
const base64Content = Buffer.from(gzipped.toString('base64'), 'binary')
699
+
700
+
// Compute CID twice to ensure consistency
701
+
const cid1 = computeCID(base64Content)
702
+
const cid2 = computeCID(base64Content)
703
+
704
+
expect(cid1).toBe(cid2)
705
+
expect(cid1).toMatch(/^bafkrei/)
706
+
})
707
+
708
+
test('should use binary encoding for base64 strings', () => {
709
+
// This test verifies we're using the correct encoding method
710
+
// For base64 strings, 'binary' encoding ensures each character becomes exactly one byte
711
+
const content = Buffer.from('Test content')
712
+
const gzipped = compressFile(content)
713
+
const base64String = gzipped.toString('base64')
714
+
715
+
// Using binary encoding (what we use in production)
716
+
const base64Content = Buffer.from(base64String, 'binary')
717
+
718
+
// Verify the length matches the base64 string length
719
+
expect(base64Content.length).toBe(base64String.length)
720
+
721
+
// Verify CID is computed correctly
722
+
const cid = computeCID(base64Content)
723
+
expect(cid).toMatch(/^bafkrei/)
724
+
})
725
+
})
726
+
727
+
describe('extractBlobMap', () => {
728
+
test('should extract blob map from flat directory structure', () => {
729
+
const mockCid = CID.parse(TEST_CID_STRING)
730
+
const mockBlob = new BlobRef(mockCid, 'text/html', 100)
731
+
732
+
const directory: Directory = {
733
+
$type: 'place.wisp.fs#directory',
734
+
type: 'directory',
735
+
entries: [
736
+
{
737
+
name: 'index.html',
738
+
node: {
739
+
$type: 'place.wisp.fs#file',
740
+
type: 'file',
741
+
blob: mockBlob,
742
+
},
743
+
},
744
+
],
745
+
}
746
+
747
+
const blobMap = extractBlobMap(directory)
748
+
749
+
expect(blobMap.size).toBe(1)
750
+
expect(blobMap.has('index.html')).toBe(true)
751
+
752
+
const entry = blobMap.get('index.html')
753
+
expect(entry?.cid).toBe(TEST_CID_STRING)
754
+
expect(entry?.blobRef).toBe(mockBlob)
755
+
})
756
+
757
+
test('should extract blob map from nested directory structure', () => {
758
+
const mockCid1 = CID.parse(TEST_CID_STRING)
759
+
const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi')
760
+
761
+
const mockBlob1 = new BlobRef(mockCid1, 'text/html', 100)
762
+
const mockBlob2 = new BlobRef(mockCid2, 'text/css', 50)
763
+
764
+
const directory: Directory = {
765
+
$type: 'place.wisp.fs#directory',
766
+
type: 'directory',
767
+
entries: [
768
+
{
769
+
name: 'index.html',
770
+
node: {
771
+
$type: 'place.wisp.fs#file',
772
+
type: 'file',
773
+
blob: mockBlob1,
774
+
},
775
+
},
776
+
{
777
+
name: 'assets',
778
+
node: {
779
+
$type: 'place.wisp.fs#directory',
780
+
type: 'directory',
781
+
entries: [
782
+
{
783
+
name: 'styles.css',
784
+
node: {
785
+
$type: 'place.wisp.fs#file',
786
+
type: 'file',
787
+
blob: mockBlob2,
788
+
},
789
+
},
790
+
],
791
+
},
792
+
},
793
+
],
794
+
}
795
+
796
+
const blobMap = extractBlobMap(directory)
797
+
798
+
expect(blobMap.size).toBe(2)
799
+
expect(blobMap.has('index.html')).toBe(true)
800
+
expect(blobMap.has('assets/styles.css')).toBe(true)
801
+
802
+
expect(blobMap.get('index.html')?.cid).toBe(TEST_CID_STRING)
803
+
expect(blobMap.get('assets/styles.css')?.cid).toBe('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi')
804
+
})
805
+
806
+
test('should handle deeply nested directory structures', () => {
807
+
const mockCid = CID.parse(TEST_CID_STRING)
808
+
const mockBlob = new BlobRef(mockCid, 'text/javascript', 200)
809
+
810
+
const directory: Directory = {
811
+
$type: 'place.wisp.fs#directory',
812
+
type: 'directory',
813
+
entries: [
814
+
{
815
+
name: 'src',
816
+
node: {
817
+
$type: 'place.wisp.fs#directory',
818
+
type: 'directory',
819
+
entries: [
820
+
{
821
+
name: 'lib',
822
+
node: {
823
+
$type: 'place.wisp.fs#directory',
824
+
type: 'directory',
825
+
entries: [
826
+
{
827
+
name: 'utils.js',
828
+
node: {
829
+
$type: 'place.wisp.fs#file',
830
+
type: 'file',
831
+
blob: mockBlob,
832
+
},
833
+
},
834
+
],
835
+
},
836
+
},
837
+
],
838
+
},
839
+
},
840
+
],
841
+
}
842
+
843
+
const blobMap = extractBlobMap(directory)
844
+
845
+
expect(blobMap.size).toBe(1)
846
+
expect(blobMap.has('src/lib/utils.js')).toBe(true)
847
+
expect(blobMap.get('src/lib/utils.js')?.cid).toBe(TEST_CID_STRING)
848
+
})
849
+
850
+
test('should handle empty directory', () => {
851
+
const directory: Directory = {
852
+
$type: 'place.wisp.fs#directory',
853
+
type: 'directory',
854
+
entries: [],
855
+
}
856
+
857
+
const blobMap = extractBlobMap(directory)
858
+
859
+
expect(blobMap.size).toBe(0)
860
+
})
861
+
862
+
test('should correctly extract CID from BlobRef instances (not plain objects)', () => {
863
+
// This test verifies the fix: AT Protocol SDK returns BlobRef instances,
864
+
// not plain objects with $type and $link properties
865
+
const mockCid = CID.parse(TEST_CID_STRING)
866
+
const mockBlob = new BlobRef(mockCid, 'application/octet-stream', 500)
867
+
868
+
const directory: Directory = {
869
+
$type: 'place.wisp.fs#directory',
870
+
type: 'directory',
871
+
entries: [
872
+
{
873
+
name: 'test.bin',
874
+
node: {
875
+
$type: 'place.wisp.fs#file',
876
+
type: 'file',
877
+
blob: mockBlob,
878
+
},
879
+
},
880
+
],
881
+
}
882
+
883
+
const blobMap = extractBlobMap(directory)
884
+
885
+
// The fix: we call .toString() on the CID instance instead of accessing $link
886
+
expect(blobMap.get('test.bin')?.cid).toBe(TEST_CID_STRING)
887
+
expect(blobMap.get('test.bin')?.blobRef.ref.toString()).toBe(TEST_CID_STRING)
888
+
})
889
+
890
+
test('should handle multiple files in same directory', () => {
891
+
const mockCid1 = CID.parse(TEST_CID_STRING)
892
+
const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi')
893
+
const mockCid3 = CID.parse('bafkreieb3ixgchss44kw7xiavnkns47emdfsqbhcdfluo3p6n3o53fl3vq')
894
+
895
+
const mockBlob1 = new BlobRef(mockCid1, 'image/png', 1000)
896
+
const mockBlob2 = new BlobRef(mockCid2, 'image/png', 2000)
897
+
const mockBlob3 = new BlobRef(mockCid3, 'image/png', 3000)
898
+
899
+
const directory: Directory = {
900
+
$type: 'place.wisp.fs#directory',
901
+
type: 'directory',
902
+
entries: [
903
+
{
904
+
name: 'images',
905
+
node: {
906
+
$type: 'place.wisp.fs#directory',
907
+
type: 'directory',
908
+
entries: [
909
+
{
910
+
name: 'logo.png',
911
+
node: {
912
+
$type: 'place.wisp.fs#file',
913
+
type: 'file',
914
+
blob: mockBlob1,
915
+
},
916
+
},
917
+
{
918
+
name: 'banner.png',
919
+
node: {
920
+
$type: 'place.wisp.fs#file',
921
+
type: 'file',
922
+
blob: mockBlob2,
923
+
},
924
+
},
925
+
{
926
+
name: 'icon.png',
927
+
node: {
928
+
$type: 'place.wisp.fs#file',
929
+
type: 'file',
930
+
blob: mockBlob3,
931
+
},
932
+
},
933
+
],
934
+
},
935
+
},
936
+
],
937
+
}
938
+
939
+
const blobMap = extractBlobMap(directory)
940
+
941
+
expect(blobMap.size).toBe(3)
942
+
expect(blobMap.has('images/logo.png')).toBe(true)
943
+
expect(blobMap.has('images/banner.png')).toBe(true)
944
+
expect(blobMap.has('images/icon.png')).toBe(true)
945
+
})
946
+
947
+
test('should handle mixed directory and file structure', () => {
948
+
const mockCid1 = CID.parse(TEST_CID_STRING)
949
+
const mockCid2 = CID.parse('bafkreiabaduc3573q6snt2xgxzpglwuaojkzflocncrh2vj5j3jykdpqhi')
950
+
const mockCid3 = CID.parse('bafkreieb3ixgchss44kw7xiavnkns47emdfsqbhcdfluo3p6n3o53fl3vq')
951
+
952
+
const directory: Directory = {
953
+
$type: 'place.wisp.fs#directory',
954
+
type: 'directory',
955
+
entries: [
956
+
{
957
+
name: 'index.html',
958
+
node: {
959
+
$type: 'place.wisp.fs#file',
960
+
type: 'file',
961
+
blob: new BlobRef(mockCid1, 'text/html', 100),
962
+
},
963
+
},
964
+
{
965
+
name: 'assets',
966
+
node: {
967
+
$type: 'place.wisp.fs#directory',
968
+
type: 'directory',
969
+
entries: [
970
+
{
971
+
name: 'styles.css',
972
+
node: {
973
+
$type: 'place.wisp.fs#file',
974
+
type: 'file',
975
+
blob: new BlobRef(mockCid2, 'text/css', 50),
976
+
},
977
+
},
978
+
],
979
+
},
980
+
},
981
+
{
982
+
name: 'README.md',
983
+
node: {
984
+
$type: 'place.wisp.fs#file',
985
+
type: 'file',
986
+
blob: new BlobRef(mockCid3, 'text/markdown', 200),
987
+
},
988
+
},
989
+
],
990
+
}
991
+
992
+
const blobMap = extractBlobMap(directory)
993
+
994
+
expect(blobMap.size).toBe(3)
995
+
expect(blobMap.has('index.html')).toBe(true)
996
+
expect(blobMap.has('assets/styles.css')).toBe(true)
997
+
expect(blobMap.has('README.md')).toBe(true)
998
+
})
999
+
})
+65
-2
src/lib/wisp-utils.ts
+65
-2
src/lib/wisp-utils.ts
···
2
2
import type { Record, Directory, File, Entry } from "../lexicons/types/place/wisp/fs";
3
3
import { validateRecord } from "../lexicons/types/place/wisp/fs";
4
4
import { gzipSync } from 'zlib';
5
+
import { CID } from 'multiformats/cid';
6
+
import { sha256 } from 'multiformats/hashes/sha2';
7
+
import * as raw from 'multiformats/codecs/raw';
8
+
import { createHash } from 'crypto';
9
+
import * as mf from 'multiformats';
5
10
6
11
export interface UploadedFile {
7
12
name: string;
···
48
53
}
49
54
50
55
/**
51
-
* Compress a file using gzip
56
+
* Compress a file using gzip with deterministic output
57
+
* Sets mtime to 0 to ensure identical content produces identical compressed output
52
58
*/
53
59
export function compressFile(content: Buffer): Buffer {
54
-
return gzipSync(content, { level: 9 });
60
+
return gzipSync(content, {
61
+
level: 9,
62
+
mtime: 0 // Fixed timestamp for deterministic compression
63
+
});
55
64
}
56
65
57
66
/**
···
65
74
const directoryMap = new Map<string, UploadedFile[]>();
66
75
67
76
for (const file of files) {
77
+
// Skip undefined/null files (defensive)
78
+
if (!file || !file.name) {
79
+
console.error('Skipping undefined or invalid file in processUploadedFiles');
80
+
continue;
81
+
}
82
+
68
83
// Remove any base folder name from the path
69
84
const normalizedPath = file.name.replace(/^[^\/]*\//, '');
70
85
const parts = normalizedPath.split('/');
···
239
254
240
255
return result;
241
256
}
257
+
258
+
/**
259
+
* Compute CID (Content Identifier) for blob content
260
+
* Uses the same algorithm as AT Protocol: CIDv1 with raw codec and SHA-256
261
+
* Based on @atproto/common/src/ipld.ts sha256RawToCid implementation
262
+
*/
263
+
export function computeCID(content: Buffer): string {
264
+
// Use node crypto to compute sha256 hash (same as AT Protocol)
265
+
const hash = createHash('sha256').update(content).digest();
266
+
// Create digest object from hash bytes
267
+
const digest = mf.digest.create(sha256.code, hash);
268
+
// Create CIDv1 with raw codec
269
+
const cid = CID.createV1(raw.code, digest);
270
+
return cid.toString();
271
+
}
272
+
273
+
/**
274
+
* Extract blob information from a directory tree
275
+
* Returns a map of file paths to their blob refs and CIDs
276
+
*/
277
+
export function extractBlobMap(
278
+
directory: Directory,
279
+
currentPath: string = ''
280
+
): Map<string, { blobRef: BlobRef; cid: string }> {
281
+
const blobMap = new Map<string, { blobRef: BlobRef; cid: string }>();
282
+
283
+
for (const entry of directory.entries) {
284
+
const fullPath = currentPath ? `${currentPath}/${entry.name}` : entry.name;
285
+
286
+
if ('type' in entry.node && entry.node.type === 'file') {
287
+
const fileNode = entry.node as File;
288
+
// AT Protocol SDK returns BlobRef class instances, not plain objects
289
+
// The ref is a CID instance that can be converted to string
290
+
if (fileNode.blob && fileNode.blob.ref) {
291
+
const cidString = fileNode.blob.ref.toString();
292
+
blobMap.set(fullPath, {
293
+
blobRef: fileNode.blob,
294
+
cid: cidString
295
+
});
296
+
}
297
+
} else if ('type' in entry.node && entry.node.type === 'directory') {
298
+
const subMap = extractBlobMap(entry.node as Directory, fullPath);
299
+
subMap.forEach((value, key) => blobMap.set(key, value));
300
+
}
301
+
}
302
+
303
+
return blobMap;
304
+
}
+130
-10
src/routes/wisp.ts
+130
-10
src/routes/wisp.ts
···
9
9
createManifest,
10
10
updateFileBlobs,
11
11
shouldCompressFile,
12
-
compressFile
12
+
compressFile,
13
+
computeCID,
14
+
extractBlobMap
13
15
} from '../lib/wisp-utils'
14
16
import { upsertSite } from '../lib/db'
15
17
import { logger } from '../lib/observability'
···
48
50
siteName: string;
49
51
files: File | File[]
50
52
};
53
+
54
+
console.log('=== UPLOAD FILES START ===');
55
+
console.log('Site name:', siteName);
56
+
console.log('Files received:', Array.isArray(files) ? files.length : 'single file');
51
57
52
58
try {
53
59
if (!siteName) {
···
106
112
107
113
// Create agent with OAuth session
108
114
const agent = new Agent((url, init) => auth.session.fetchHandler(url, init))
115
+
console.log('Agent created for DID:', auth.did);
116
+
117
+
// Try to fetch existing record to enable incremental updates
118
+
let existingBlobMap = new Map<string, { blobRef: any; cid: string }>();
119
+
console.log('Attempting to fetch existing record...');
120
+
try {
121
+
const rkey = siteName;
122
+
const existingRecord = await agent.com.atproto.repo.getRecord({
123
+
repo: auth.did,
124
+
collection: 'place.wisp.fs',
125
+
rkey: rkey
126
+
});
127
+
console.log('Existing record found!');
128
+
129
+
if (existingRecord.data.value && typeof existingRecord.data.value === 'object' && 'root' in existingRecord.data.value) {
130
+
const manifest = existingRecord.data.value as any;
131
+
existingBlobMap = extractBlobMap(manifest.root);
132
+
console.log(`Found existing manifest with ${existingBlobMap.size} files for incremental update`);
133
+
logger.info(`Found existing manifest with ${existingBlobMap.size} files for incremental update`);
134
+
}
135
+
} catch (error: any) {
136
+
console.log('No existing record found or error:', error?.message || error);
137
+
// Record doesn't exist yet, this is a new site
138
+
if (error?.status !== 400 && error?.error !== 'RecordNotFound') {
139
+
logger.warn('Failed to fetch existing record, proceeding with full upload', error);
140
+
}
141
+
}
109
142
110
143
// Convert File objects to UploadedFile format
111
144
// Elysia gives us File objects directly, handle both single file and array
···
113
146
const uploadedFiles: UploadedFile[] = [];
114
147
const skippedFiles: Array<{ name: string; reason: string }> = [];
115
148
116
-
149
+
console.log('Processing files, count:', fileArray.length);
117
150
118
151
for (let i = 0; i < fileArray.length; i++) {
119
152
const file = fileArray[i];
153
+
console.log(`Processing file ${i + 1}/${fileArray.length}:`, file.name, file.size, 'bytes');
120
154
121
155
// Skip files that are too large (limit to 100MB per file)
122
156
const maxSize = MAX_FILE_SIZE; // 100MB
···
135
169
// Compress and base64 encode ALL files
136
170
const compressedContent = compressFile(originalContent);
137
171
// Base64 encode the gzipped content to prevent PDS content sniffing
138
-
const base64Content = Buffer.from(compressedContent.toString('base64'), 'utf-8');
172
+
// Convert base64 string to bytes using binary encoding (each char becomes exactly one byte)
173
+
// This is what PDS receives and computes CID on
174
+
const base64Content = Buffer.from(compressedContent.toString('base64'), 'binary');
139
175
const compressionRatio = (compressedContent.length / originalContent.length * 100).toFixed(1);
176
+
console.log(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`);
140
177
logger.info(`Compressing ${file.name}: ${originalContent.length} -> ${compressedContent.length} bytes (${compressionRatio}%), base64: ${base64Content.length} bytes`);
141
178
142
179
uploadedFiles.push({
143
180
name: file.name,
144
-
content: base64Content,
181
+
content: base64Content, // This is the gzipped+base64 content that will be uploaded and CID-computed
145
182
mimeType: originalMimeType,
146
183
size: base64Content.length,
147
184
compressed: true,
···
206
243
}
207
244
208
245
// Process files into directory structure
209
-
const { directory, fileCount } = processUploadedFiles(uploadedFiles);
246
+
console.log('Processing uploaded files into directory structure...');
247
+
console.log('uploadedFiles array length:', uploadedFiles.length);
248
+
console.log('uploadedFiles contents:', uploadedFiles.map((f, i) => `${i}: ${f?.name || 'UNDEFINED'}`));
210
249
211
-
// Upload files as blobs in parallel
250
+
// Filter out any undefined/null/invalid entries (defensive)
251
+
const validUploadedFiles = uploadedFiles.filter((f, i) => {
252
+
if (!f) {
253
+
console.error(`Filtering out undefined/null file at index ${i}`);
254
+
return false;
255
+
}
256
+
if (!f.name) {
257
+
console.error(`Filtering out file with no name at index ${i}:`, f);
258
+
return false;
259
+
}
260
+
if (!f.content) {
261
+
console.error(`Filtering out file with no content at index ${i}:`, f.name);
262
+
return false;
263
+
}
264
+
return true;
265
+
});
266
+
if (validUploadedFiles.length !== uploadedFiles.length) {
267
+
console.warn(`Filtered out ${uploadedFiles.length - validUploadedFiles.length} invalid files`);
268
+
}
269
+
console.log('validUploadedFiles length:', validUploadedFiles.length);
270
+
271
+
const { directory, fileCount } = processUploadedFiles(validUploadedFiles);
272
+
console.log('Directory structure created, file count:', fileCount);
273
+
274
+
// Upload files as blobs in parallel (or reuse existing blobs with matching CIDs)
275
+
console.log('Starting blob upload/reuse phase...');
212
276
// For compressed files, we upload as octet-stream and store the original MIME type in metadata
213
277
// For text/html files, we also use octet-stream as a workaround for PDS image pipeline issues
214
-
const uploadPromises = uploadedFiles.map(async (file, i) => {
278
+
const uploadPromises = validUploadedFiles.map(async (file, i) => {
215
279
try {
280
+
// Skip undefined files (shouldn't happen after filter, but defensive)
281
+
if (!file || !file.name) {
282
+
console.error(`ERROR: Undefined file at index ${i} in validUploadedFiles!`);
283
+
throw new Error(`Undefined file at index ${i}`);
284
+
}
285
+
286
+
// Compute CID for this file to check if it already exists
287
+
// Note: file.content is already gzipped+base64 encoded
288
+
const fileCID = computeCID(file.content);
289
+
290
+
// Normalize the file path for comparison (remove base folder prefix like "cobblemon/")
291
+
const normalizedPath = file.name.replace(/^[^\/]*\//, '');
292
+
293
+
// Check if we have an existing blob with the same CID
294
+
// Try both the normalized path and the full path
295
+
const existingBlob = existingBlobMap.get(normalizedPath) || existingBlobMap.get(file.name);
296
+
297
+
if (existingBlob && existingBlob.cid === fileCID) {
298
+
// Reuse existing blob - no need to upload
299
+
logger.info(`[File Upload] Reusing existing blob for: ${file.name} (CID: ${fileCID})`);
300
+
301
+
return {
302
+
result: {
303
+
hash: existingBlob.cid,
304
+
blobRef: existingBlob.blobRef,
305
+
...(file.compressed && {
306
+
encoding: 'gzip' as const,
307
+
mimeType: file.originalMimeType || file.mimeType,
308
+
base64: true
309
+
})
310
+
},
311
+
filePath: file.name,
312
+
sentMimeType: file.mimeType,
313
+
returnedMimeType: existingBlob.blobRef.mimeType,
314
+
reused: true
315
+
};
316
+
}
317
+
318
+
// File is new or changed - upload it
216
319
// If compressed, always upload as octet-stream
217
320
// Otherwise, workaround: PDS incorrectly processes text/html through image pipeline
218
321
const uploadMimeType = file.compressed || file.mimeType.startsWith('text/html')
···
220
323
: file.mimeType;
221
324
222
325
const compressionInfo = file.compressed ? ' (gzipped)' : '';
223
-
logger.info(`[File Upload] Uploading file: ${file.name} (original: ${file.mimeType}, sending as: ${uploadMimeType}, ${file.size} bytes${compressionInfo})`);
326
+
logger.info(`[File Upload] Uploading new/changed file: ${file.name} (original: ${file.mimeType}, sending as: ${uploadMimeType}, ${file.size} bytes${compressionInfo}, CID: ${fileCID})`);
224
327
225
328
const uploadResult = await agent.com.atproto.repo.uploadBlob(
226
329
file.content,
···
244
347
},
245
348
filePath: file.name,
246
349
sentMimeType: file.mimeType,
247
-
returnedMimeType: returnedBlobRef.mimeType
350
+
returnedMimeType: returnedBlobRef.mimeType,
351
+
reused: false
248
352
};
249
353
} catch (uploadError) {
250
354
logger.error('Upload failed for file', uploadError);
···
255
359
// Wait for all uploads to complete
256
360
const uploadedBlobs = await Promise.all(uploadPromises);
257
361
362
+
// Count reused vs uploaded blobs
363
+
const reusedCount = uploadedBlobs.filter(b => (b as any).reused).length;
364
+
const uploadedCount = uploadedBlobs.filter(b => !(b as any).reused).length;
365
+
console.log(`Blob statistics: ${reusedCount} reused, ${uploadedCount} uploaded, ${uploadedBlobs.length} total`);
366
+
logger.info(`Blob statistics: ${reusedCount} reused, ${uploadedCount} uploaded, ${uploadedBlobs.length} total`);
367
+
258
368
// Extract results and file paths in correct order
259
369
const uploadResults: FileUploadResult[] = uploadedBlobs.map(blob => blob.result);
260
370
const filePaths: string[] = uploadedBlobs.map(blob => blob.filePath);
261
371
262
372
// Update directory with file blobs
373
+
console.log('Updating directory with blob references...');
263
374
const updatedDirectory = updateFileBlobs(directory, uploadResults, filePaths);
264
375
265
376
// Create manifest
377
+
console.log('Creating manifest...');
266
378
const manifest = createManifest(siteName, updatedDirectory, fileCount);
379
+
console.log('Manifest created successfully');
267
380
268
381
// Use site name as rkey
269
382
const rkey = siteName;
270
383
271
384
let record;
272
385
try {
386
+
console.log('Putting record to PDS with rkey:', rkey);
273
387
record = await agent.com.atproto.repo.putRecord({
274
388
repo: auth.did,
275
389
collection: 'place.wisp.fs',
276
390
rkey: rkey,
277
391
record: manifest
278
392
});
393
+
console.log('Record successfully created on PDS:', record.data.uri);
279
394
} catch (putRecordError: any) {
395
+
console.error('FAILED to create record on PDS:', putRecordError);
280
396
logger.error('Failed to create record on PDS', putRecordError);
281
397
282
398
throw putRecordError;
···
292
408
fileCount,
293
409
siteName,
294
410
skippedFiles,
295
-
uploadedCount: uploadedFiles.length
411
+
uploadedCount: validUploadedFiles.length
296
412
};
297
413
414
+
console.log('=== UPLOAD FILES COMPLETE ===');
298
415
return result;
299
416
} catch (error) {
417
+
console.error('=== UPLOAD ERROR ===');
418
+
console.error('Error details:', error);
419
+
console.error('Stack trace:', error instanceof Error ? error.stack : 'N/A');
300
420
logger.error('Upload error', error, {
301
421
message: error instanceof Error ? error.message : 'Unknown error',
302
422
name: error instanceof Error ? error.name : undefined