Code and data for arewedecentralizedyet.online and related projects
Merge branch 'main' of ssh://codeberg.org/ricci/are-we-decentralized-yet
+662
-70
+38
-22
helpers/update-datafile.py
+38
-22
helpers/update-datafile.py
···
111
111
112
112
current_terms = shannon_terms_by_host(current_csv)
113
113
previous_terms = shannon_terms_by_host(previous_csv)
114
+
current_counts = user_counts_by_host(current_csv)
115
+
previous_counts = user_counts_by_host(previous_csv)
114
116
all_hosts = set(current_terms) | set(previous_terms)
115
-
best_pos_host = None
116
-
best_pos_diff = 0.0
117
-
best_neg_host = None
118
-
best_neg_diff = 0.0
117
+
diffs = []
119
118
for host in sorted(all_hosts):
120
119
term_diff = current_terms.get(host, 0.0) - previous_terms.get(host, 0.0)
121
-
if term_diff > 0 and (best_pos_host is None or term_diff > best_pos_diff):
122
-
best_pos_host = host
123
-
best_pos_diff = term_diff
124
-
if term_diff < 0 and (best_neg_host is None or term_diff < best_neg_diff):
125
-
best_neg_host = host
126
-
best_neg_diff = term_diff
120
+
user_diff = current_counts.get(host, 0) - previous_counts.get(host, 0)
121
+
diffs.append((host, term_diff, user_diff))
122
+
top_increase = [
123
+
{
124
+
"host": host,
125
+
"change": round(term_diff, 6),
126
+
"user_change": user_diff,
127
+
}
128
+
for host, term_diff, user_diff in sorted(diffs, key=lambda item: item[1], reverse=True)
129
+
if term_diff > 0
130
+
][:10]
131
+
top_decrease = [
132
+
{
133
+
"host": host,
134
+
"change": round(term_diff, 6),
135
+
"user_change": user_diff,
136
+
}
137
+
for host, term_diff, user_diff in sorted(diffs, key=lambda item: item[1])
138
+
if term_diff < 0
139
+
][:10]
127
140
data.setdefault("trends", {}).setdefault(key, {}).setdefault(period, {})["shannon_contrib"] = {
128
-
"increase": {
129
-
"host": best_pos_host,
130
-
"change": round(best_pos_diff, 6),
131
-
},
132
-
"decrease": {
133
-
"host": best_neg_host,
134
-
"change": round(best_neg_diff, 6),
135
-
},
141
+
"increase": top_increase,
142
+
"decrease": top_decrease,
136
143
}
137
144
138
145
139
146
def shannon_terms_by_host(csv_path):
140
-
rows = load_csv_rows(csv_path)
141
-
rows = filter_rows(rows)
142
-
extracted = [extract_domain_counts(row) for row in rows]
143
-
combined = combine_rows(extracted)
147
+
combined = combined_counts(csv_path)
144
148
if not combined:
145
149
return {}
146
150
total = sum(item["count"] for item in combined)
···
149
153
counts = [item["count"] for item in combined]
150
154
terms = _shannon(counts, return_terms=True)
151
155
return {item["domain"]: term for item, term in zip(combined, terms)}
156
+
157
+
158
+
def user_counts_by_host(csv_path):
159
+
combined = combined_counts(csv_path)
160
+
return {item["domain"]: item["count"] for item in combined}
161
+
162
+
163
+
def combined_counts(csv_path):
164
+
rows = load_csv_rows(csv_path)
165
+
rows = filter_rows(rows)
166
+
extracted = [extract_domain_counts(row) for row in rows]
167
+
return combine_rows(extracted)
152
168
153
169
154
170
def load_csv_rows(csv_path):
+624
-48
www/data.js
+624
-48
www/data.js
···
266
266
"weekly": {
267
267
"shannon": -0.0107,
268
268
"shannon_contrib": {
269
-
"increase": {
270
-
"host": "theforkiverse.com",
271
-
"change": 0.013765
272
-
},
273
-
"decrease": {
274
-
"host": "gram.social",
275
-
"change": -0.003856
276
-
}
269
+
"increase": [
270
+
{
271
+
"host": "theforkiverse.com",
272
+
"change": 0.013765,
273
+
"user_change": 3682
274
+
},
275
+
{
276
+
"host": "mstdn.z.org",
277
+
"change": 0.01278,
278
+
"user_change": 2348
279
+
},
280
+
{
281
+
"host": "pixelfed.social",
282
+
"change": 0.00449,
283
+
"user_change": 4898
284
+
},
285
+
{
286
+
"host": "hawirkitabak.com",
287
+
"change": 0.003304,
288
+
"user_change": 496
289
+
},
290
+
{
291
+
"host": "pixelfed.au",
292
+
"change": 0.002132,
293
+
"user_change": 398
294
+
},
295
+
{
296
+
"host": "pixelfed.tokyo",
297
+
"change": 0.002105,
298
+
"user_change": 402
299
+
},
300
+
{
301
+
"host": "shitposter.world",
302
+
"change": 0.001318,
303
+
"user_change": 170
304
+
},
305
+
{
306
+
"host": "write.rlp.schule",
307
+
"change": 0.000824,
308
+
"user_change": 141
309
+
},
310
+
{
311
+
"host": "urbanists.social",
312
+
"change": 0.000705,
313
+
"user_change": 128
314
+
},
315
+
{
316
+
"host": "pixelfed.de.",
317
+
"change": 0.000693,
318
+
"user_change": 274
319
+
}
320
+
],
321
+
"decrease": [
322
+
{
323
+
"host": "gram.social",
324
+
"change": -0.003856,
325
+
"user_change": -1000
326
+
},
327
+
{
328
+
"host": "fed.brid.gy",
329
+
"change": -0.001909,
330
+
"user_change": 262
331
+
},
332
+
{
333
+
"host": "micro.blog",
334
+
"change": -0.001887,
335
+
"user_change": -388
336
+
},
337
+
{
338
+
"host": "yourpix.au",
339
+
"change": -0.001069,
340
+
"user_change": -140
341
+
},
342
+
{
343
+
"host": "lemmy.world",
344
+
"change": -0.001049,
345
+
"user_change": -81
346
+
},
347
+
{
348
+
"host": "sociale.network",
349
+
"change": -0.001048,
350
+
"user_change": -152
351
+
},
352
+
{
353
+
"host": "pleroma.z.org",
354
+
"change": -0.001,
355
+
"user_change": -173
356
+
},
357
+
{
358
+
"host": "openbiblio.social",
359
+
"change": -0.000938,
360
+
"user_change": -148
361
+
},
362
+
{
363
+
"host": "organica.social",
364
+
"change": -0.000832,
365
+
"user_change": -137
366
+
},
367
+
{
368
+
"host": "st.fdel.moe",
369
+
"change": -0.000826,
370
+
"user_change": -115
371
+
}
372
+
]
277
373
}
278
374
},
279
375
"monthly": {
280
376
"shannon": -0.0107,
281
377
"shannon_contrib": {
282
-
"increase": {
283
-
"host": "theforkiverse.com",
284
-
"change": 0.013765
285
-
},
286
-
"decrease": {
287
-
"host": "gram.social",
288
-
"change": -0.003856
289
-
}
378
+
"increase": [
379
+
{
380
+
"host": "theforkiverse.com",
381
+
"change": 0.013765,
382
+
"user_change": 3682
383
+
},
384
+
{
385
+
"host": "mstdn.z.org",
386
+
"change": 0.01278,
387
+
"user_change": 2348
388
+
},
389
+
{
390
+
"host": "pixelfed.social",
391
+
"change": 0.00449,
392
+
"user_change": 4898
393
+
},
394
+
{
395
+
"host": "hawirkitabak.com",
396
+
"change": 0.003304,
397
+
"user_change": 496
398
+
},
399
+
{
400
+
"host": "pixelfed.au",
401
+
"change": 0.002132,
402
+
"user_change": 398
403
+
},
404
+
{
405
+
"host": "pixelfed.tokyo",
406
+
"change": 0.002105,
407
+
"user_change": 402
408
+
},
409
+
{
410
+
"host": "shitposter.world",
411
+
"change": 0.001318,
412
+
"user_change": 170
413
+
},
414
+
{
415
+
"host": "write.rlp.schule",
416
+
"change": 0.000824,
417
+
"user_change": 141
418
+
},
419
+
{
420
+
"host": "urbanists.social",
421
+
"change": 0.000705,
422
+
"user_change": 128
423
+
},
424
+
{
425
+
"host": "pixelfed.de.",
426
+
"change": 0.000693,
427
+
"user_change": 274
428
+
}
429
+
],
430
+
"decrease": [
431
+
{
432
+
"host": "gram.social",
433
+
"change": -0.003856,
434
+
"user_change": -1000
435
+
},
436
+
{
437
+
"host": "fed.brid.gy",
438
+
"change": -0.001909,
439
+
"user_change": 262
440
+
},
441
+
{
442
+
"host": "micro.blog",
443
+
"change": -0.001887,
444
+
"user_change": -388
445
+
},
446
+
{
447
+
"host": "yourpix.au",
448
+
"change": -0.001069,
449
+
"user_change": -140
450
+
},
451
+
{
452
+
"host": "lemmy.world",
453
+
"change": -0.001049,
454
+
"user_change": -81
455
+
},
456
+
{
457
+
"host": "sociale.network",
458
+
"change": -0.001048,
459
+
"user_change": -152
460
+
},
461
+
{
462
+
"host": "pleroma.z.org",
463
+
"change": -0.001,
464
+
"user_change": -173
465
+
},
466
+
{
467
+
"host": "openbiblio.social",
468
+
"change": -0.000938,
469
+
"user_change": -148
470
+
},
471
+
{
472
+
"host": "organica.social",
473
+
"change": -0.000832,
474
+
"user_change": -137
475
+
},
476
+
{
477
+
"host": "st.fdel.moe",
478
+
"change": -0.000826,
479
+
"user_change": -115
480
+
}
481
+
]
290
482
}
291
483
}
292
484
},
···
294
486
"weekly": {
295
487
"shannon": -0.0011,
296
488
"shannon_contrib": {
297
-
"increase": {
298
-
"host": "skystack.xyz",
299
-
"change": 6.3e-05
300
-
},
301
-
"decrease": {
302
-
"host": "randomly-generated.ngrok-free.app",
303
-
"change": -0.000262
304
-
}
489
+
"increase": [
490
+
{
491
+
"host": "skystack.xyz",
492
+
"change": 6.3e-05,
493
+
"user_change": 60
494
+
},
495
+
{
496
+
"host": "pds.sprk.so",
497
+
"change": 3.6e-05,
498
+
"user_change": 29
499
+
},
500
+
{
501
+
"host": "pds.ridgeway.dev",
502
+
"change": 3.5e-05,
503
+
"user_change": 16
504
+
},
505
+
{
506
+
"host": "northsky.social",
507
+
"change": 3.3e-05,
508
+
"user_change": 22
509
+
},
510
+
{
511
+
"host": "pds.rip",
512
+
"change": 2.3e-05,
513
+
"user_change": 12
514
+
},
515
+
{
516
+
"host": "dev-pds.periwinkle.social",
517
+
"change": 2.1e-05,
518
+
"user_change": 9
519
+
},
520
+
{
521
+
"host": "pds.quack.space",
522
+
"change": 1.9e-05,
523
+
"user_change": 9
524
+
},
525
+
{
526
+
"host": "pds-staging.ridgeway.network",
527
+
"change": 1.6e-05,
528
+
"user_change": 7
529
+
},
530
+
{
531
+
"host": "pds2.dolciss.net",
532
+
"change": 1.6e-05,
533
+
"user_change": 7
534
+
},
535
+
{
536
+
"host": "psrd.club",
537
+
"change": 1.6e-05,
538
+
"user_change": 7
539
+
}
540
+
],
541
+
"decrease": [
542
+
{
543
+
"host": "randomly-generated.ngrok-free.app",
544
+
"change": -0.000262,
545
+
"user_change": 0
546
+
},
547
+
{
548
+
"host": "tngl.sh",
549
+
"change": -0.000227,
550
+
"user_change": -149
551
+
},
552
+
{
553
+
"host": "blacksky.app",
554
+
"change": -0.000153,
555
+
"user_change": -77
556
+
},
557
+
{
558
+
"host": ".host.bsky.network",
559
+
"change": -0.000142,
560
+
"user_change": 94062
561
+
},
562
+
{
563
+
"host": "gems.xyz",
564
+
"change": -0.000133,
565
+
"user_change": -79
566
+
},
567
+
{
568
+
"host": "gumby.social",
569
+
"change": -7.9e-05,
570
+
"user_change": -43
571
+
},
572
+
{
573
+
"host": "atproto.brid.gy",
574
+
"change": -5.3e-05,
575
+
"user_change": 316
576
+
},
577
+
{
578
+
"host": "zio.blue",
579
+
"change": -4.3e-05,
580
+
"user_change": -22
581
+
},
582
+
{
583
+
"host": "bsky.bestofmodels.blog",
584
+
"change": -3.7e-05,
585
+
"user_change": -20
586
+
},
587
+
{
588
+
"host": "pds.bsky.yinzcloud.net",
589
+
"change": -3.6e-05,
590
+
"user_change": -18
591
+
}
592
+
]
305
593
}
306
594
},
307
595
"monthly": {
308
596
"shannon": -0.0009,
309
597
"shannon_contrib": {
310
-
"increase": {
311
-
"host": "x.mt.social",
312
-
"change": 0.00022
313
-
},
314
-
"decrease": {
315
-
"host": "randomly-generated.ngrok-free.app",
316
-
"change": -0.000399
317
-
}
598
+
"increase": [
599
+
{
600
+
"host": "x.mt.social",
601
+
"change": 0.00022,
602
+
"user_change": 161
603
+
},
604
+
{
605
+
"host": "pds.sprk.so",
606
+
"change": 9.8e-05,
607
+
"user_change": 71
608
+
},
609
+
{
610
+
"host": "climateai.org",
611
+
"change": 7.9e-05,
612
+
"user_change": 44
613
+
},
614
+
{
615
+
"host": "skystack.xyz",
616
+
"change": 5.6e-05,
617
+
"user_change": 62
618
+
},
619
+
{
620
+
"host": "northsky.social",
621
+
"change": 4.6e-05,
622
+
"user_change": 30
623
+
},
624
+
{
625
+
"host": "pds.ridgeway.dev",
626
+
"change": 3.5e-05,
627
+
"user_change": 16
628
+
},
629
+
{
630
+
"host": "sds-eu-west4.test.certified.app",
631
+
"change": 2.8e-05,
632
+
"user_change": 14
633
+
},
634
+
{
635
+
"host": "pds.rip",
636
+
"change": 2.6e-05,
637
+
"user_change": 14
638
+
},
639
+
{
640
+
"host": "dev-pds.periwinkle.social",
641
+
"change": 2.1e-05,
642
+
"user_change": 9
643
+
},
644
+
{
645
+
"host": "pds-dev.heart-land.io",
646
+
"change": 2e-05,
647
+
"user_change": 9
648
+
}
649
+
],
650
+
"decrease": [
651
+
{
652
+
"host": "randomly-generated.ngrok-free.app",
653
+
"change": -0.000399,
654
+
"user_change": 0
655
+
},
656
+
{
657
+
"host": "tngl.sh",
658
+
"change": -0.000217,
659
+
"user_change": -137
660
+
},
661
+
{
662
+
"host": "blacksky.app",
663
+
"change": -0.000148,
664
+
"user_change": -40
665
+
},
666
+
{
667
+
"host": ".host.bsky.network",
668
+
"change": -0.000144,
669
+
"user_change": 141622
670
+
},
671
+
{
672
+
"host": "gems.xyz",
673
+
"change": -0.000136,
674
+
"user_change": -79
675
+
},
676
+
{
677
+
"host": "atproto.brid.gy",
678
+
"change": -0.000125,
679
+
"user_change": 421
680
+
},
681
+
{
682
+
"host": "gumby.social",
683
+
"change": -7.5e-05,
684
+
"user_change": -40
685
+
},
686
+
{
687
+
"host": "zio.blue",
688
+
"change": -4.2e-05,
689
+
"user_change": -21
690
+
},
691
+
{
692
+
"host": "pds.bsky.yinzcloud.net",
693
+
"change": -3.9e-05,
694
+
"user_change": -19
695
+
},
696
+
{
697
+
"host": "bsky.bestofmodels.blog",
698
+
"change": -3.6e-05,
699
+
"user_change": -18
700
+
}
701
+
]
318
702
}
319
703
}
320
704
},
···
322
706
"weekly": {
323
707
"shannon": -0.0004,
324
708
"shannon_contrib": {
325
-
"increase": {
326
-
"host": "gitlabext.wsl.ch",
327
-
"change": 1e-06
328
-
},
329
-
"decrease": {
330
-
"host": "gitlab.com",
331
-
"change": -0.000147
332
-
}
709
+
"increase": [
710
+
{
711
+
"host": "gitlabext.wsl.ch",
712
+
"change": 1e-06,
713
+
"user_change": 21
714
+
},
715
+
{
716
+
"host": "git.tstarr.us",
717
+
"change": 0.0,
718
+
"user_change": 4
719
+
},
720
+
{
721
+
"host": "gitea.com",
722
+
"change": 0.0,
723
+
"user_change": 30
724
+
},
725
+
{
726
+
"host": "gitlab.huma-num.fr",
727
+
"change": 0.0,
728
+
"user_change": 6
729
+
},
730
+
{
731
+
"host": "gitlab.esss.lu.se",
732
+
"change": 0.0,
733
+
"user_change": 17
734
+
},
735
+
{
736
+
"host": "gitea.michalczyk.pro",
737
+
"change": 0.0,
738
+
"user_change": 2
739
+
},
740
+
{
741
+
"host": "git.toradex.com",
742
+
"change": 0.0,
743
+
"user_change": 2
744
+
},
745
+
{
746
+
"host": "git.leximpact.dev",
747
+
"change": 0.0,
748
+
"user_change": 2
749
+
},
750
+
{
751
+
"host": "gitlab.opencode.de",
752
+
"change": 0.0,
753
+
"user_change": 5
754
+
},
755
+
{
756
+
"host": "gitlab.dsi.universite-paris-saclay.fr",
757
+
"change": 0.0,
758
+
"user_change": 14
759
+
}
760
+
],
761
+
"decrease": [
762
+
{
763
+
"host": "gitlab.com",
764
+
"change": -0.000147,
765
+
"user_change": 1066
766
+
},
767
+
{
768
+
"host": "github",
769
+
"change": -8.1e-05,
770
+
"user_change": 929061
771
+
},
772
+
{
773
+
"host": "bitbucket",
774
+
"change": -7.5e-05,
775
+
"user_change": 2539
776
+
},
777
+
{
778
+
"host": "packagist",
779
+
"change": -2e-05,
780
+
"user_change": 37
781
+
},
782
+
{
783
+
"host": "main",
784
+
"change": -1.1e-05,
785
+
"user_change": -1
786
+
},
787
+
{
788
+
"host": "gitorious",
789
+
"change": -8e-06,
790
+
"user_change": 0
791
+
},
792
+
{
793
+
"host": "launchpad",
794
+
"change": -8e-06,
795
+
"user_change": 4
796
+
},
797
+
{
798
+
"host": "googlecode",
799
+
"change": -6e-06,
800
+
"user_change": 0
801
+
},
802
+
{
803
+
"host": "salsa.debian.org",
804
+
"change": -5e-06,
805
+
"user_change": 8
806
+
},
807
+
{
808
+
"host": "git.drupalcode.org",
809
+
"change": -4e-06,
810
+
"user_change": 0
811
+
}
812
+
]
333
813
}
334
814
},
335
815
"monthly": {
336
816
"shannon": -0.0019,
337
817
"shannon_contrib": {
338
-
"increase": {
339
-
"host": "git.fossgalaxy.com",
340
-
"change": 7e-06
341
-
},
342
-
"decrease": {
343
-
"host": "gitlab.com",
344
-
"change": -0.000676
345
-
}
818
+
"increase": [
819
+
{
820
+
"host": "git.fossgalaxy.com",
821
+
"change": 7e-06,
822
+
"user_change": 145
823
+
},
824
+
{
825
+
"host": "git.kernel.org",
826
+
"change": 4e-06,
827
+
"user_change": 114
828
+
},
829
+
{
830
+
"host": "gitlab.dsi.universite-paris-saclay.fr",
831
+
"change": 3e-06,
832
+
"user_change": 142
833
+
},
834
+
{
835
+
"host": "git.windmaker.net",
836
+
"change": 2e-06,
837
+
"user_change": 48
838
+
},
839
+
{
840
+
"host": "git.jordan.im",
841
+
"change": 2e-06,
842
+
"user_change": 32
843
+
},
844
+
{
845
+
"host": "git.finf.uni-hannover.de",
846
+
"change": 1e-06,
847
+
"user_change": 24
848
+
},
849
+
{
850
+
"host": "gitlabext.wsl.ch",
851
+
"change": 1e-06,
852
+
"user_change": 21
853
+
},
854
+
{
855
+
"host": "gitlab.esss.lu.se",
856
+
"change": 1e-06,
857
+
"user_change": 81
858
+
},
859
+
{
860
+
"host": "gitea.com",
861
+
"change": 0.0,
862
+
"user_change": 123
863
+
},
864
+
{
865
+
"host": "code.peren.fr",
866
+
"change": 0.0,
867
+
"user_change": 7
868
+
}
869
+
],
870
+
"decrease": [
871
+
{
872
+
"host": "gitlab.com",
873
+
"change": -0.000676,
874
+
"user_change": 4577
875
+
},
876
+
{
877
+
"host": "github",
878
+
"change": -0.00037,
879
+
"user_change": 4206993
880
+
},
881
+
{
882
+
"host": "bitbucket",
883
+
"change": -0.000332,
884
+
"user_change": 12223
885
+
},
886
+
{
887
+
"host": "packagist",
888
+
"change": -9e-05,
889
+
"user_change": 310
890
+
},
891
+
{
892
+
"host": "main",
893
+
"change": -5.1e-05,
894
+
"user_change": -1
895
+
},
896
+
{
897
+
"host": "gitorious",
898
+
"change": -3.6e-05,
899
+
"user_change": 0
900
+
},
901
+
{
902
+
"host": "launchpad",
903
+
"change": -3.5e-05,
904
+
"user_change": 14
905
+
},
906
+
{
907
+
"host": "googlecode",
908
+
"change": -2.8e-05,
909
+
"user_change": 0
910
+
},
911
+
{
912
+
"host": "salsa.debian.org",
913
+
"change": -2.2e-05,
914
+
"user_change": 62
915
+
},
916
+
{
917
+
"host": "git.drupalcode.org",
918
+
"change": -1.9e-05,
919
+
"user_change": 0
920
+
}
921
+
]
346
922
}
347
923
}
348
924
}