this repo has no description
Increase batch size
+11
-12
+2
-2
scripts/crawl_follows.py
+2
-2
scripts/crawl_follows.py
···
11
11
from atproto import exceptions as at_exceptions
12
12
import pandas as pd
13
13
14
-
from utils import RateLimit
14
+
from utils import RateLimit, BSKY_API_LIMIT
15
15
16
16
logger = logging.getLogger(__name__)
17
17
logger.setLevel(logging.INFO)
···
95
95
# Try to only send 10 requests a second
96
96
batch_count = 1
97
97
fail_count = 0
98
-
rate_limiter = RateLimit(BATCH_SIZE)
98
+
rate_limiter = RateLimit(BSKY_API_LIMIT)
99
99
while len(to_explore):
100
100
batch = to_explore[:BATCH_SIZE]
101
101
to_explore = to_explore[BATCH_SIZE:]
+3
-4
scripts/get_follows.py
+3
-4
scripts/get_follows.py
···
10
10
from atproto import exceptions as at_exceptions
11
11
from atproto_client.models.app.bsky.graph.follow import Record as FollowRecord
12
12
13
-
from utils import load_checkpoint, get_accounts, RateLimit
13
+
from utils import load_checkpoint, get_accounts, RateLimit, BSKY_API_LIMIT
14
14
15
15
logger = logging.getLogger(__name__)
16
16
logger.setLevel(logging.INFO)
···
26
26
logger.addHandler(console_handler)
27
27
28
28
29
-
BATCH_SIZE = 10
30
-
FOLLOWER_THRESHOLD = 150
29
+
BATCH_SIZE = 100
31
30
32
31
33
32
async def get_all_follows(
···
87
86
# Get all follows for accounts
88
87
batch_count = 0
89
88
fail_count = 0
90
-
rate_limiter = RateLimit(BATCH_SIZE)
89
+
rate_limiter = RateLimit(BSKY_API_LIMIT)
91
90
for i in range(0, len(accts), BATCH_SIZE):
92
91
batch = [acct for acct, _ in accts[i : i + BATCH_SIZE]]
93
92
for result in asyncio.as_completed(
+2
-3
scripts/get_likes.py
+2
-3
scripts/get_likes.py
···
11
11
from atproto import exceptions as at_exceptions
12
12
from atproto_client.models.app.bsky.feed.like import Record as LikeRecord
13
13
14
-
from utils import load_checkpoint, get_accounts, RateLimit
14
+
from utils import load_checkpoint, get_accounts, RateLimit, BSKY_API_LIMIT
15
15
16
16
logger = logging.getLogger(__name__)
17
17
logger.setLevel(logging.INFO)
···
28
28
29
29
30
30
BATCH_SIZE = 10
31
-
FOLLOWER_THRESHOLD = 150
32
31
REQUIRED_ENV = ("BSKY_USER", "BSKY_APP_PW")
33
32
34
33
···
104
103
# Get all posts for accounts
105
104
batch_count = 0
106
105
fail_count = 0
107
-
rate_limiter = RateLimit(BATCH_SIZE)
106
+
rate_limiter = RateLimit(BSKY_API_LIMIT)
108
107
for i in range(0, len(accts), BATCH_SIZE):
109
108
batch = [acct for acct, _ in accts[i : i + BATCH_SIZE]]
110
109
for result in asyncio.as_completed(
+2
-3
scripts/get_posts.py
+2
-3
scripts/get_posts.py
···
12
12
from atproto import exceptions as at_exceptions
13
13
from atproto_client.models.app.bsky.feed.defs import FeedViewPost
14
14
15
-
from utils import get_accounts, load_checkpoint, RateLimit
15
+
from utils import get_accounts, load_checkpoint, RateLimit, BSKY_API_LIMIT
16
16
17
17
logger = logging.getLogger(__name__)
18
18
logger.setLevel(logging.INFO)
···
29
29
30
30
31
31
BATCH_SIZE = 10
32
-
FOLLOWER_THRESHOLD = 150
33
32
REQUIRED_ENV = ("BSKY_USER", "BSKY_APP_PW")
34
33
35
34
···
163
162
# Get all posts for accounts
164
163
batch_count = 0
165
164
fail_count = 0
166
-
rate_limiter = RateLimit(BATCH_SIZE)
165
+
rate_limiter = RateLimit(BSKY_API_LIMIT)
167
166
for i in range(0, len(accts), BATCH_SIZE):
168
167
batch = [acct for acct, _ in accts[i : i + BATCH_SIZE]]
169
168
for result in asyncio.as_completed(