-193

prototyping/redis-noodling-2.ipynb

reviewed

··· 1 1 - { 2 2 - "cells": [ 3 3 - { 4 4 - "cell_type": "code", 5 5 - "execution_count": null, 6 6 - "id": "f2fd24f9", 7 7 - "metadata": {}, 8 8 - "outputs": [ 9 9 - { 10 10 - "ename": "ConnectionError", 11 11 - "evalue": "Error 61 connecting to localhost:6379. Connection refused.", 12 12 - "output_type": "error", 13 13 - "traceback": [ 14 14 - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", 15 15 - "\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)", 16 16 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:378\u001b[39m, in \u001b[36mAbstractConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 377\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m378\u001b[39m sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mretry\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcall_with_retry\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 379\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_connect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mdisconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43merror\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 380\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 381\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.timeout:\n", 17 17 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/retry.py:62\u001b[39m, in \u001b[36mRetry.call_with_retry\u001b[39m\u001b[34m(self, do, fail)\u001b[39m\n\u001b[32m 61\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m62\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mdo\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 63\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;28mself\u001b[39m._supported_errors \u001b[38;5;28;01mas\u001b[39;00m error:\n", 18 18 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:379\u001b[39m, in \u001b[36mAbstractConnection.connect.<locals>.<lambda>\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 377\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 378\u001b[39m sock = \u001b[38;5;28mself\u001b[39m.retry.call_with_retry(\n\u001b[32m--> \u001b[39m\u001b[32m379\u001b[39m \u001b[38;5;28;01mlambda\u001b[39;00m: \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_connect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, \u001b[38;5;28;01mlambda\u001b[39;00m error: \u001b[38;5;28mself\u001b[39m.disconnect(error)\n\u001b[32m 380\u001b[39m )\n\u001b[32m 381\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.timeout:\n", 19 19 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:764\u001b[39m, in \u001b[36mConnection._connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 763\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m err \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m764\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 765\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33msocket.getaddrinfo returned an empty list\u001b[39m\u001b[33m\"\u001b[39m)\n", 20 20 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:752\u001b[39m, in \u001b[36mConnection._connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 751\u001b[39m \u001b[38;5;66;03m# connect\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m752\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msocket_address\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 754\u001b[39m \u001b[38;5;66;03m# set the socket_timeout now that we're connected\u001b[39;00m\n", 21 21 - "\u001b[31mConnectionRefusedError\u001b[39m: [Errno 61] Connection refused", 22 22 - "\nDuring handling of the above exception, another exception occurred:\n", 23 23 - "\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)", 24 24 - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 4\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01matdata\u001b[39;00m\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01matdata\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mlocal\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mal\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m4\u001b[39m local_index = \u001b[43mal\u001b[49m\u001b[43m.\u001b[49m\u001b[43mIndex\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 5\u001b[39m x: al.IndexEntry = local_index.list()[\u001b[32m0\u001b[39m]\n", 25 25 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/src/atdata/local.py:61\u001b[39m, in \u001b[36mIndex.__init__\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 55\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"TODO\"\"\"\u001b[39;00m\n\u001b[32m 56\u001b[39m \u001b[38;5;66;03m##\u001b[39;00m\n\u001b[32m 57\u001b[39m \n\u001b[32m 58\u001b[39m \u001b[38;5;66;03m# ...\u001b[39;00m\n\u001b[32m 59\u001b[39m \n\u001b[32m 60\u001b[39m \u001b[38;5;66;03m# Needed before we can do anything with redis-om queries\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m61\u001b[39m \u001b[43mMigrator\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", 26 26 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis_om/model/migrations/migrator.py:163\u001b[39m, in \u001b[36mMigrator.run\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 160\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mrun\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[32m 161\u001b[39m \u001b[38;5;66;03m# TODO: Migration history\u001b[39;00m\n\u001b[32m 162\u001b[39m \u001b[38;5;66;03m# TODO: Dry run with output\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m163\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mdetect_migrations\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 164\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m migration \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.migrations:\n\u001b[32m 165\u001b[39m migration.run()\n", 27 27 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis_om/model/migrations/migrator.py:118\u001b[39m, in \u001b[36mMigrator.detect_migrations\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 115\u001b[39m current_hash = hashlib.sha1(schema.encode(\u001b[33m\"\u001b[39m\u001b[33mutf-8\u001b[39m\u001b[33m\"\u001b[39m)).hexdigest() \u001b[38;5;66;03m# nosec\u001b[39;00m\n\u001b[32m 117\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m118\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mft\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mMeta\u001b[49m\u001b[43m.\u001b[49m\u001b[43mindex_name\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43minfo\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 119\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m redis.ResponseError:\n\u001b[32m 120\u001b[39m \u001b[38;5;28mself\u001b[39m.migrations.append(\n\u001b[32m 121\u001b[39m IndexMigration(\n\u001b[32m 122\u001b[39m name,\n\u001b[32m (...)\u001b[39m\u001b[32m 128\u001b[39m )\n\u001b[32m 129\u001b[39m )\n", 28 28 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/commands/search/commands.py:452\u001b[39m, in \u001b[36mSearchCommands.info\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 444\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34minfo\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[32m 445\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 446\u001b[39m \u001b[33;03m Get info an stats about the the current index, including the number of\u001b[39;00m\n\u001b[32m 447\u001b[39m \u001b[33;03m documents, memory consumption, etc\u001b[39;00m\n\u001b[32m 448\u001b[39m \n\u001b[32m 449\u001b[39m \u001b[33;03m For more information see `FT.INFO <https://redis.io/commands/ft.info>`_.\u001b[39;00m\n\u001b[32m 450\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m452\u001b[39m res = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mexecute_command\u001b[49m\u001b[43m(\u001b[49m\u001b[43mINFO_CMD\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mindex_name\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 453\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._parse_results(INFO_CMD, res)\n", 29 29 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/client.py:605\u001b[39m, in \u001b[36mRedis.execute_command\u001b[39m\u001b[34m(self, *args, **options)\u001b[39m\n\u001b[32m 604\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mexecute_command\u001b[39m(\u001b[38;5;28mself\u001b[39m, *args, **options):\n\u001b[32m--> \u001b[39m\u001b[32m605\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_execute_command\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m)\u001b[49m\n", 30 30 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/client.py:611\u001b[39m, in \u001b[36mRedis._execute_command\u001b[39m\u001b[34m(self, *args, **options)\u001b[39m\n\u001b[32m 609\u001b[39m pool = \u001b[38;5;28mself\u001b[39m.connection_pool\n\u001b[32m 610\u001b[39m command_name = args[\u001b[32m0\u001b[39m]\n\u001b[32m--> \u001b[39m\u001b[32m611\u001b[39m conn = \u001b[38;5;28mself\u001b[39m.connection \u001b[38;5;129;01mor\u001b[39;00m \u001b[43mpool\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_connection\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 613\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._single_connection_client:\n\u001b[32m 614\u001b[39m \u001b[38;5;28mself\u001b[39m.single_connection_lock.acquire()\n", 31 31 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/utils.py:183\u001b[39m, in \u001b[36mdeprecated_args.<locals>.decorator.<locals>.wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 178\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m arg \u001b[38;5;129;01min\u001b[39;00m provided_args:\n\u001b[32m 179\u001b[39m warn_deprecated_arg_usage(\n\u001b[32m 180\u001b[39m arg, func.\u001b[34m__name__\u001b[39m, reason, version, stacklevel=\u001b[32m3\u001b[39m\n\u001b[32m 181\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m183\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", 32 32 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:1483\u001b[39m, in \u001b[36mConnectionPool.get_connection\u001b[39m\u001b[34m(self, command_name, *keys, **options)\u001b[39m\n\u001b[32m 1479\u001b[39m \u001b[38;5;28mself\u001b[39m._in_use_connections.add(connection)\n\u001b[32m 1481\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 1482\u001b[39m \u001b[38;5;66;03m# ensure this connection is connected to Redis\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1483\u001b[39m \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1484\u001b[39m \u001b[38;5;66;03m# connections that the pool provides should be ready to send\u001b[39;00m\n\u001b[32m 1485\u001b[39m \u001b[38;5;66;03m# a command. if not, the connection was either returned to the\u001b[39;00m\n\u001b[32m 1486\u001b[39m \u001b[38;5;66;03m# pool before all data has been read or the socket has been\u001b[39;00m\n\u001b[32m 1487\u001b[39m \u001b[38;5;66;03m# closed. either way, reconnect and verify everything is good.\u001b[39;00m\n\u001b[32m 1488\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n", 33 33 - "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:384\u001b[39m, in \u001b[36mAbstractConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 382\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33mTimeout connecting to server\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 383\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m384\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(\u001b[38;5;28mself\u001b[39m._error_message(e))\n\u001b[32m 386\u001b[39m \u001b[38;5;28mself\u001b[39m._sock = sock\n\u001b[32m 387\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n", 34 34 - "\u001b[31mConnectionError\u001b[39m: Error 61 connecting to localhost:6379. Connection refused." 35 35 - ] 36 36 - } 37 37 - ], 38 38 - "source": [ 39 39 - "import atdata\n", 40 40 - "import atdata._local_dev as al\n", 41 41 - "\n", 42 42 - "local_index = al.Index()\n", 43 43 - "x: al.IndexEntry = local_index.list()[0]" 44 44 - ] 45 45 - }, 46 46 - { 47 47 - "cell_type": "code", 48 48 - "execution_count": 4, 49 49 - "id": "c968a5c7", 50 50 - "metadata": {}, 51 51 - "outputs": [ 52 52 - { 53 53 - "data": { 54 54 - "text/plain": [ 55 55 - "{'$id': 'https://schema.dev/fake-schema.schema.json',\n", 56 56 - " '$schema': 'http://json-schema.org/draft-07/schema#',\n", 57 57 - " 'title': 'fake-schema',\n", 58 58 - " 'description': 'Blue Blah',\n", 59 59 - " 'type': 'object',\n", 60 60 - " 'properties': {'property_a': {'default': 5, 'type': 'integer'},\n", 61 61 - " 'property_b': {'type': 'string'}}}" 62 62 - ] 63 63 - }, 64 64 - "execution_count": 4, 65 65 - "metadata": {}, 66 66 - "output_type": "execute_result" 67 67 - } 68 68 - ], 69 69 - "source": [ 70 70 - "x.sample_schema.json_schema" 71 71 - ] 72 72 - }, 73 73 - { 74 74 - "cell_type": "code", 75 75 - "execution_count": 6, 76 76 - "id": "d4761aa8", 77 77 - "metadata": {}, 78 78 - "outputs": [], 79 79 - "source": [ 80 80 - "from pydantic import create_model, Field" 81 81 - ] 82 82 - }, 83 83 - { 84 84 - "cell_type": "code", 85 85 - "execution_count": 7, 86 86 - "id": "3282fd1d", 87 87 - "metadata": {}, 88 88 - "outputs": [], 89 89 - "source": [ 90 90 - "json_schema = x.sample_schema.json_schema\n", 91 91 - "\n", 92 92 - "# Map JSON Schema types and constraints to Pydantic fields\n", 93 93 - "fields = {}\n", 94 94 - "for prop_name, prop_details in json_schema[\"properties\"].items():\n", 95 95 - " field_type = None\n", 96 96 - " if prop_details[\"type\"] == \"string\":\n", 97 97 - " if prop_details.get(\"format\") == \"email\":\n", 98 98 - " field_type = str # Pydantic's EmailStr can be used for stronger validation\n", 99 99 - " else:\n", 100 100 - " field_type = str\n", 101 101 - " elif prop_details[\"type\"] == \"integer\":\n", 102 102 - " field_type = int\n", 103 103 - " # Add more type mappings as needed\n", 104 104 - "\n", 105 105 - " field_args = {}\n", 106 106 - " if \"maxLength\" in prop_details:\n", 107 107 - " field_args[\"max_length\"] = prop_details[\"maxLength\"]\n", 108 108 - " if \"minimum\" in prop_details:\n", 109 109 - " field_args[\"ge\"] = prop_details[\"minimum\"]\n", 110 110 - " # Add more constraint mappings (e.g., pattern, minLength, maximum, etc.)\n", 111 111 - "\n", 112 112 - " if prop_name in json_schema.get(\"required\", []):\n", 113 113 - " fields[prop_name] = (field_type, Field(**field_args))\n", 114 114 - " else:\n", 115 115 - " fields[prop_name] = (field_type, Field(None, **field_args)) # Optional field" 116 116 - ] 117 117 - }, 118 118 - { 119 119 - "cell_type": "code", 120 120 - "execution_count": 27, 121 121 - "id": "f9dc5833", 122 122 - "metadata": {}, 123 123 - "outputs": [], 124 124 - "source": [ 125 125 - "_FakeSchema = create_model( 'FakeSchema', **fields )\n", 126 126 - "class FakeSchema( _FakeSchema, atdata.PackableSample ):\n", 127 127 - " pass" 128 128 - ] 129 129 - }, 130 130 - { 131 131 - "cell_type": "code", 132 132 - "execution_count": 37, 133 133 - "id": "41ed6eee", 134 134 - "metadata": {}, 135 135 - "outputs": [], 136 136 - "source": [ 137 137 - "x = FakeSchema( property_a = 2, property_b = 'hello' )" 138 138 - ] 139 139 - }, 140 140 - { 141 141 - "cell_type": "code", 142 142 - "execution_count": 38, 143 143 - "id": "1f6f3f97", 144 144 - "metadata": {}, 145 145 - "outputs": [], 146 146 - "source": [ 147 147 - "x_prime = FakeSchema.from_bytes( x.as_wds['msgpack'] )" 148 148 - ] 149 149 - }, 150 150 - { 151 151 - "cell_type": "code", 152 152 - "execution_count": 39, 153 153 - "id": "bd139ab0", 154 154 - "metadata": {}, 155 155 - "outputs": [ 156 156 - { 157 157 - "data": { 158 158 - "text/plain": [ 159 159 - "FakeSchema(property_a=2, property_b='hello')" 160 160 - ] 161 161 - }, 162 162 - "execution_count": 39, 163 163 - "metadata": {}, 164 164 - "output_type": "execute_result" 165 165 - } 166 166 - ], 167 167 - "source": [ 168 168 - "x_prime" 169 169 - ] 170 170 - } 171 171 - ], 172 172 - "metadata": { 173 173 - "kernelspec": { 174 174 - "display_name": "atdata", 175 175 - "language": "python", 176 176 - "name": "python3" 177 177 - }, 178 178 - "language_info": { 179 179 - "codemirror_mode": { 180 180 - "name": "ipython", 181 181 - "version": 3 182 182 - }, 183 183 - "file_extension": ".py", 184 184 - "mimetype": "text/x-python", 185 185 - "name": "python", 186 186 - "nbconvert_exporter": "python", 187 187 - "pygments_lexer": "ipython3", 188 188 - "version": "3.12.11" 189 189 - } 190 190 - }, 191 191 - "nbformat": 4, 192 192 - "nbformat_minor": 5 193 193 - }

-132

prototyping/redis-noodling-3.ipynb

reviewed

··· 1 1 - { 2 2 - "cells": [ 3 3 - { 4 4 - "cell_type": "code", 5 5 - "execution_count": 1, 6 6 - "id": "256e5c04", 7 7 - "metadata": {}, 8 8 - "outputs": [], 9 9 - "source": [ 10 10 - "import atdata\n", 11 11 - "import atdata.local as al" 12 12 - ] 13 13 - }, 14 14 - { 15 15 - "cell_type": "code", 16 16 - "execution_count": 5, 17 17 - "id": "d9df3d99", 18 18 - "metadata": {}, 19 19 - "outputs": [], 20 20 - "source": [ 21 21 - "from redis import Redis\n", 22 22 - "\n", 23 23 - "test_url = \"rediss://default:AWWlAAIncDEzZGM2ZGM2NjJiMWE0N2EwYjI0ZWM1ZTcxYWIyYjIxZXAxMjYwMjE@settled-moose-26021.upstash.io:6379\"\n", 24 24 - "r = Redis.from_url( test_url )" 25 25 - ] 26 26 - }, 27 27 - { 28 28 - "cell_type": "code", 29 29 - "execution_count": 6, 30 30 - "id": "a98b9f90", 31 31 - "metadata": {}, 32 32 - "outputs": [ 33 33 - { 34 34 - "data": { 35 35 - "text/plain": [ 36 36 - "True" 37 37 - ] 38 38 - }, 39 39 - "execution_count": 6, 40 40 - "metadata": {}, 41 41 - "output_type": "execute_result" 42 42 - } 43 43 - ], 44 44 - "source": [ 45 45 - "r.set( 'baz', 'bar' )" 46 46 - ] 47 47 - }, 48 48 - { 49 49 - "cell_type": "code", 50 50 - "execution_count": 8, 51 51 - "id": "47b08315", 52 52 - "metadata": {}, 53 53 - "outputs": [ 54 54 - { 55 55 - "data": { 56 56 - "text/plain": [ 57 57 - "b'bar'" 58 58 - ] 59 59 - }, 60 60 - "execution_count": 8, 61 61 - "metadata": {}, 62 62 - "output_type": "execute_result" 63 63 - } 64 64 - ], 65 65 - "source": [ 66 66 - "r.get( 'baz' )" 67 67 - ] 68 68 - }, 69 69 - { 70 70 - "cell_type": "code", 71 71 - "execution_count": 9, 72 72 - "id": "b1a1e2f2", 73 73 - "metadata": {}, 74 74 - "outputs": [], 75 75 - "source": [ 76 76 - "redis_index = al.Index( redis = r )" 77 77 - ] 78 78 - }, 79 79 - { 80 80 - "cell_type": "code", 81 81 - "execution_count": 10, 82 82 - "id": "10aab6ce", 83 83 - "metadata": {}, 84 84 - "outputs": [ 85 85 - { 86 86 - "data": { 87 87 - "text/plain": [ 88 88 - "[]" 89 89 - ] 90 90 - }, 91 91 - "execution_count": 10, 92 92 - "metadata": {}, 93 93 - "output_type": "execute_result" 94 94 - } 95 95 - ], 96 96 - "source": [ 97 97 - "redis_index.list()" 98 98 - ] 99 99 - }, 100 100 - { 101 101 - "cell_type": "code", 102 102 - "execution_count": null, 103 103 - "id": "b5f66fda", 104 104 - "metadata": {}, 105 105 - "outputs": [], 106 106 - "source": [ 107 107 - "ds = atdata.Dataset( )" 108 108 - ] 109 109 - } 110 110 - ], 111 111 - "metadata": { 112 112 - "kernelspec": { 113 113 - "display_name": "atdata", 114 114 - "language": "python", 115 115 - "name": "python3" 116 116 - }, 117 117 - "language_info": { 118 118 - "codemirror_mode": { 119 119 - "name": "ipython", 120 120 - "version": 3 121 121 - }, 122 122 - "file_extension": ".py", 123 123 - "mimetype": "text/x-python", 124 124 - "name": "python", 125 125 - "nbconvert_exporter": "python", 126 126 - "pygments_lexer": "ipython3", 127 127 - "version": "3.12.11" 128 128 - } 129 129 - }, 130 130 - "nbformat": 4, 131 131 - "nbformat_minor": 5 132 132 - }

-203

prototyping/redis-noodling-4.ipynb

reviewed

··· 1 1 - { 2 2 - "cells": [ 3 3 - { 4 4 - "cell_type": "code", 5 5 - "execution_count": 1, 6 6 - "id": "bcb96018", 7 7 - "metadata": {}, 8 8 - "outputs": [], 9 9 - "source": [ 10 10 - "from redis import Redis\n", 11 11 - "\n", 12 12 - "import atdata.local as al" 13 13 - ] 14 14 - }, 15 15 - { 16 16 - "cell_type": "code", 17 17 - "execution_count": 2, 18 18 - "id": "d1631435", 19 19 - "metadata": {}, 20 20 - "outputs": [], 21 21 - "source": [ 22 22 - "test_url = \"rediss://default:AWWlAAIncDEzZGM2ZGM2NjJiMWE0N2EwYjI0ZWM1ZTcxYWIyYjIxZXAxMjYwMjE@settled-moose-26021.upstash.io:6379\"\n", 23 23 - "test_redis = Redis.from_url( test_url )" 24 24 - ] 25 25 - }, 26 26 - { 27 27 - "cell_type": "code", 28 28 - "execution_count": 3, 29 29 - "id": "5c9ee2e2", 30 30 - "metadata": {}, 31 31 - "outputs": [], 32 32 - "source": [ 33 33 - "local_repo = al.Repo(\n", 34 34 - " s3_credentials = '.credentials/r2-analysis-hive.env',\n", 35 35 - " hive_path = 'analysis-hive/test-hive/',\n", 36 36 - " redis = test_redis,\n", 37 37 - ")" 38 38 - ] 39 39 - }, 40 40 - { 41 41 - "cell_type": "code", 42 42 - "execution_count": 4, 43 43 - "id": "4425dc0d", 44 44 - "metadata": {}, 45 45 - "outputs": [], 46 46 - "source": [ 47 47 - "import atdata\n", 48 48 - "import webdataset as wds\n", 49 49 - "from dataclasses import dataclass\n", 50 50 - "from numpy.typing import NDArray\n", 51 51 - "\n", 52 52 - "@dataclass\n", 53 53 - "class TestSample( atdata.PackableSample ):\n", 54 54 - " identifier: int\n", 55 55 - " values: NDArray" 56 56 - ] 57 57 - }, 58 58 - { 59 59 - "cell_type": "code", 60 60 - "execution_count": 5, 61 61 - "id": "8c2ed4b2", 62 62 - "metadata": {}, 63 63 - "outputs": [ 64 64 - { 65 65 - "name": "stdout", 66 66 - "output_type": "stream", 67 67 - "text": [ 68 68 - "# writing data/test-000000.tar 0 0.0 GB 0\n", 69 69 - "# writing data/test-000001.tar 100 0.2 GB 100\n", 70 70 - "# writing data/test-000002.tar 100 0.2 GB 200\n", 71 71 - "# writing data/test-000003.tar 100 0.2 GB 300\n", 72 72 - "# writing data/test-000004.tar 100 0.2 GB 400\n", 73 73 - "# writing data/test-000005.tar 100 0.2 GB 500\n", 74 74 - "# writing data/test-000006.tar 100 0.2 GB 600\n", 75 75 - "# writing data/test-000007.tar 100 0.2 GB 700\n", 76 76 - "# writing data/test-000008.tar 100 0.2 GB 800\n", 77 77 - "# writing data/test-000009.tar 100 0.2 GB 900\n" 78 78 - ] 79 79 - } 80 80 - ], 81 81 - "source": [ 82 82 - "import numpy as np\n", 83 83 - "n = 950\n", 84 84 - "\n", 85 85 - "import os\n", 86 86 - "os.makedirs( 'data', exist_ok = True )\n", 87 87 - "\n", 88 88 - "with wds.writer.ShardWriter( 'data/test-%06d.tar', maxcount = 100 ) as sink:\n", 89 89 - " for i in range( n ):\n", 90 90 - " new_sample = TestSample(\n", 91 91 - " identifier = i + 1,\n", 92 92 - " values = np.random.normal( size = (1024, 256) ),\n", 93 93 - " )\n", 94 94 - " sink.write( new_sample.as_wds )" 95 95 - ] 96 96 - }, 97 97 - { 98 98 - "cell_type": "code", 99 99 - "execution_count": 6, 100 100 - "id": "ba794054", 101 101 - "metadata": {}, 102 102 - "outputs": [ 103 103 - { 104 104 - "name": "stdout", 105 105 - "output_type": "stream", 106 106 - "text": [ 107 107 - "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000000.tar 0 0.0 GB 0\n", 108 108 - "Copying file to s3 ... done.\n", 109 109 - "Deleting local cache file ... done.\n", 110 110 - "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000001.tar 150 0.3 GB 150\n", 111 111 - "Copying file to s3 ... done.\n", 112 112 - "Deleting local cache file ... done.\n", 113 113 - "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000002.tar 150 0.3 GB 300\n", 114 114 - "Copying file to s3 ... done.\n", 115 115 - "Deleting local cache file ... done.\n", 116 116 - "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000003.tar 150 0.3 GB 450\n", 117 117 - "Copying file to s3 ... done.\n", 118 118 - "Deleting local cache file ... done.\n", 119 119 - "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000004.tar 150 0.3 GB 600\n", 120 120 - "Copying file to s3 ... done.\n", 121 121 - "Deleting local cache file ... done.\n", 122 122 - "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000005.tar 150 0.3 GB 750\n", 123 123 - "Copying file to s3 ... done.\n", 124 124 - "Deleting local cache file ... done.\n", 125 125 - "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000006.tar 150 0.3 GB 900\n", 126 126 - "Copying file to s3 ... done.\n", 127 127 - "Deleting local cache file ... done.\n" 128 128 - ] 129 129 - }, 130 130 - { 131 131 - "data": { 132 132 - "text/plain": [ 133 133 - "(BasicIndexEntry(wds_url='analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--{000000..-00001}.tar', sample_kind='__main__.TestSample', metadata_url='analysis-hive/test-hive/metadata/atdata-metadata--bf546505-3e70-44d4-ab1f-9729ad25ce8c.msgpack', uuid='bf546505-3e70-44d4-ab1f-9729ad25ce8c'),\n", 134 134 - " <atdata.dataset.Dataset at 0x111daa9f0>)" 135 135 - ] 136 136 - }, 137 137 - "execution_count": 6, 138 138 - "metadata": {}, 139 139 - "output_type": "execute_result" 140 140 - } 141 141 - ], 142 142 - "source": [ 143 143 - "new_dataset = atdata.Dataset[TestSample]( 'data/test-{000000..000009}.tar' )\n", 144 144 - "local_repo.insert( new_dataset,\n", 145 145 - " maxcount = 150,\n", 146 146 - " cache_local = True,\n", 147 147 - ")" 148 148 - ] 149 149 - }, 150 150 - { 151 151 - "cell_type": "code", 152 152 - "execution_count": 8, 153 153 - "id": "31f5f666", 154 154 - "metadata": {}, 155 155 - "outputs": [], 156 156 - "source": [ 157 157 - "x = local_repo.index.list()[0]" 158 158 - ] 159 159 - }, 160 160 - { 161 161 - "cell_type": "code", 162 162 - "execution_count": 7, 163 163 - "id": "85924ef6", 164 164 - "metadata": {}, 165 165 - "outputs": [ 166 166 - { 167 167 - "name": "stdout", 168 168 - "output_type": "stream", 169 169 - "text": [ 170 170 - "BasicIndexEntry(wds_url='analysis-hive/test-hive/atdata--138f45b5-3ba4-46b7-9521-4930093e2ffc--{000000..000000}.tar', sample_kind='atdata.local.T', metadata_url='analysis-hive/test-hive/metadata/atdata-metadata--138f45b5-3ba4-46b7-9521-4930093e2ffc.msgpack', uuid='138f45b5-3ba4-46b7-9521-4930093e2ffc')\n", 171 171 - "BasicIndexEntry(wds_url='analysis-hive/test-hive/atdata--66854cd0-1c3e-4ad2-a757-e100d566a878--{000000..000006}.tar', sample_kind='__main__.TestSample', metadata_url='analysis-hive/test-hive/metadata/atdata-metadata--66854cd0-1c3e-4ad2-a757-e100d566a878.msgpack', uuid='66854cd0-1c3e-4ad2-a757-e100d566a878')\n", 172 172 - "BasicIndexEntry(wds_url='analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--{000000..-00001}.tar', sample_kind='__main__.TestSample', metadata_url='analysis-hive/test-hive/metadata/atdata-metadata--bf546505-3e70-44d4-ab1f-9729ad25ce8c.msgpack', uuid='bf546505-3e70-44d4-ab1f-9729ad25ce8c')\n" 173 173 - ] 174 174 - } 175 175 - ], 176 176 - "source": [ 177 177 - "for e in local_repo.index.entries:\n", 178 178 - " print( e )" 179 179 - ] 180 180 - } 181 181 - ], 182 182 - "metadata": { 183 183 - "kernelspec": { 184 184 - "display_name": "atdata", 185 185 - "language": "python", 186 186 - "name": "python3" 187 187 - }, 188 188 - "language_info": { 189 189 - "codemirror_mode": { 190 190 - "name": "ipython", 191 191 - "version": 3 192 192 - }, 193 193 - "file_extension": ".py", 194 194 - "mimetype": "text/x-python", 195 195 - "name": "python", 196 196 - "nbconvert_exporter": "python", 197 197 - "pygments_lexer": "ipython3", 198 198 - "version": "3.12.11" 199 199 - } 200 200 - }, 201 201 - "nbformat": 4, 202 202 - "nbformat_minor": 5 203 203 - }

-396

prototyping/redis-noodling.ipynb

reviewed

··· 1 1 - { 2 2 - "cells": [ 3 3 - { 4 4 - "cell_type": "code", 5 5 - "execution_count": null, 6 6 - "id": "177dfb94", 7 7 - "metadata": {}, 8 8 - "outputs": [], 9 9 - "source": [ 10 10 - "import atdata\n", 11 11 - "from dataclasses import dataclass\n", 12 12 - "import atdata._local_dev as al" 13 13 - ] 14 14 - }, 15 15 - { 16 16 - "cell_type": "code", 17 17 - "execution_count": 2, 18 18 - "id": "f9cb805d", 19 19 - "metadata": {}, 20 20 - "outputs": [], 21 21 - "source": [ 22 22 - "import redis" 23 23 - ] 24 24 - }, 25 25 - { 26 26 - "cell_type": "code", 27 27 - "execution_count": 3, 28 28 - "id": "48e3c969", 29 29 - "metadata": {}, 30 30 - "outputs": [ 31 31 - { 32 32 - "data": { 33 33 - "text/plain": [ 34 34 - "[b':atdata.local.IndexEntry:01KB1CEARQXTTZDP1A8211645V',\n", 35 35 - " b':atdata.local.IndexEntry:01KB1D7R2SG9EDX5EHEFEC8R2N',\n", 36 36 - " b':atdata.local.IndexEntry:01KB1DDAW9HQ99XFF8FPFQ5AVT',\n", 37 37 - " b':atdata.local.IndexEntry:01KB1DMKTFGHY0ZH3VTNJ1H9XG',\n", 38 38 - " b':atdata.local.IndexEntry:index:hash',\n", 39 39 - " b':atdata.local.IndexEntry:01KB1CEHX24E9VDGG23DSMDZAF']" 40 40 - ] 41 41 - }, 42 42 - "execution_count": 3, 43 43 - "metadata": {}, 44 44 - "output_type": "execute_result" 45 45 - } 46 46 - ], 47 47 - "source": [ 48 48 - "pool = redis.ConnectionPool(host='localhost', port=6379, db=0)\n", 49 49 - "r = redis.Redis(connection_pool=pool)\n", 50 50 - "\n", 51 51 - "# Check specific keys\n", 52 52 - "r.keys('*')" 53 53 - ] 54 54 - }, 55 55 - { 56 56 - "cell_type": "code", 57 57 - "execution_count": 4, 58 58 - "id": "69d8b045", 59 59 - "metadata": {}, 60 60 - "outputs": [], 61 61 - "source": [ 62 62 - "local_index = al.Index()" 63 63 - ] 64 64 - }, 65 65 - { 66 66 - "cell_type": "code", 67 67 - "execution_count": 5, 68 68 - "id": "b157c142", 69 69 - "metadata": {}, 70 70 - "outputs": [], 71 71 - "source": [ 72 72 - "@dataclass\n", 73 73 - "class TestSample( atdata.PackableSample ):\n", 74 74 - " identifier: int\n", 75 75 - " name: str\n", 76 76 - "\n", 77 77 - "dataset = atdata.Dataset[TestSample]( '/data/test.tar' )" 78 78 - ] 79 79 - }, 80 80 - { 81 81 - "cell_type": "code", 82 82 - "execution_count": 6, 83 83 - "id": "6cc845f4", 84 84 - "metadata": {}, 85 85 - "outputs": [ 86 86 - { 87 87 - "data": { 88 88 - "text/plain": [ 89 89 - "IndexEntry(pk='01KB1DTP0QJZGV49J1339T9256', wds_url='/data/test.tar', sample_schema=SampleSchema(pk='01KB1DTP0PQ4XKV2N5HHQC3GWK', identifier='74deedc1-fbbb-4b11-9f99-cd73d3257983', json_schema={'$id': 'https://schema.dev/fake-schema.schema.json', '$schema': 'http://json-schema.org/draft-07/schema#', 'title': 'fake-schema', 'description': 'Blue Blah', 'type': 'object', 'properties': {'property_a': {'default': 5, 'type': 'integer'}, 'property_b': {'type': 'string'}}}), metadata_schema=None)" 90 90 - ] 91 91 - }, 92 92 - "execution_count": 6, 93 93 - "metadata": {}, 94 94 - "output_type": "execute_result" 95 95 - } 96 96 - ], 97 97 - "source": [ 98 98 - "local_index.add( dataset )" 99 99 - ] 100 100 - }, 101 101 - { 102 102 - "cell_type": "code", 103 103 - "execution_count": 7, 104 104 - "id": "ae382e86", 105 105 - "metadata": {}, 106 106 - "outputs": [], 107 107 - "source": [ 108 108 - "x: al.IndexEntry = local_index.list()[0]" 109 109 - ] 110 110 - }, 111 111 - { 112 112 - "cell_type": "code", 113 113 - "execution_count": 8, 114 114 - "id": "7edf8718", 115 115 - "metadata": {}, 116 116 - "outputs": [ 117 117 - { 118 118 - "data": { 119 119 - "text/plain": [ 120 120 - "{'$id': 'https://schema.dev/fake-schema.schema.json',\n", 121 121 - " '$schema': 'http://json-schema.org/draft-07/schema#',\n", 122 122 - " 'title': 'fake-schema',\n", 123 123 - " 'description': 'Blue Blah',\n", 124 124 - " 'type': 'object',\n", 125 125 - " 'properties': {'property_a': {'default': 5, 'type': 'integer'},\n", 126 126 - " 'property_b': {'type': 'string'}}}" 127 127 - ] 128 128 - }, 129 129 - "execution_count": 8, 130 130 - "metadata": {}, 131 131 - "output_type": "execute_result" 132 132 - } 133 133 - ], 134 134 - "source": [ 135 135 - "x.sample_schema.json_schema" 136 136 - ] 137 137 - }, 138 138 - { 139 139 - "cell_type": "code", 140 140 - "execution_count": 46, 141 141 - "id": "618ec152", 142 142 - "metadata": {}, 143 143 - "outputs": [], 144 144 - "source": [ 145 145 - "def _test_init( self, a: int, b: str ):\n", 146 146 - " self.property_a = a\n", 147 147 - " self.property_b = b\n", 148 148 - "\n", 149 149 - "TestType = type( 'TestType', (object,), {\n", 150 150 - " '__init__': _test_init,\n", 151 151 - " 'property_a': 4,\n", 152 152 - " '__annotations__': {\n", 153 153 - " 'property_a': int,\n", 154 154 - " 'property_b': str,\n", 155 155 - " }\n", 156 156 - "} )" 157 157 - ] 158 158 - }, 159 159 - { 160 160 - "cell_type": "code", 161 161 - "execution_count": null, 162 162 - "id": "d7f52957", 163 163 - "metadata": {}, 164 164 - "outputs": [], 165 165 - "source": [ 166 166 - "TestType.property_a" 167 167 - ] 168 168 - }, 169 169 - { 170 170 - "cell_type": "code", 171 171 - "execution_count": 49, 172 172 - "id": "5cba0e62", 173 173 - "metadata": {}, 174 174 - "outputs": [], 175 175 - "source": [ 176 176 - "x = TestType( 4, 'shark' )" 177 177 - ] 178 178 - }, 179 179 - { 180 180 - "cell_type": "code", 181 181 - "execution_count": 50, 182 182 - "id": "7209e369", 183 183 - "metadata": {}, 184 184 - "outputs": [ 185 185 - { 186 186 - "data": { 187 187 - "text/plain": [ 188 188 - "4" 189 189 - ] 190 190 - }, 191 191 - "execution_count": 50, 192 192 - "metadata": {}, 193 193 - "output_type": "execute_result" 194 194 - } 195 195 - ], 196 196 - "source": [ 197 197 - "x.property_a" 198 198 - ] 199 199 - }, 200 200 - { 201 201 - "cell_type": "code", 202 202 - "execution_count": null, 203 203 - "id": "3e88a7e1", 204 204 - "metadata": {}, 205 205 - "outputs": [], 206 206 - "source": [] 207 207 - }, 208 208 - { 209 209 - "cell_type": "code", 210 210 - "execution_count": 9, 211 211 - "id": "2156e6cd", 212 212 - "metadata": {}, 213 213 - "outputs": [ 214 214 - { 215 215 - "data": { 216 216 - "text/plain": [ 217 217 - "True" 218 218 - ] 219 219 - }, 220 220 - "execution_count": 9, 221 221 - "metadata": {}, 222 222 - "output_type": "execute_result" 223 223 - } 224 224 - ], 225 225 - "source": [ 226 226 - "from schemamodels import SchemaModelFactory\n", 227 227 - "\n", 228 228 - "factory = SchemaModelFactory()\n", 229 229 - "factory.register( x.sample_schema.json_schema )" 230 230 - ] 231 231 - }, 232 232 - { 233 233 - "cell_type": "code", 234 234 - "execution_count": 31, 235 235 - "id": "f26d2901", 236 236 - "metadata": {}, 237 237 - "outputs": [ 238 238 - { 239 239 - "ename": "TypeError", 240 240 - "evalue": "cannot inherit non-frozen dataclass from a frozen one", 241 241 - "output_type": "error", 242 242 - "traceback": [ 243 243 - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", 244 244 - "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", 245 245 - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[31]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mschemamodels\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdynamic\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m FakeSchema\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m \u001b[38;5;129;43m@dataclass\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[43mfrozen\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4\u001b[39m \u001b[38;5;28;43;01mclass\u001b[39;49;00m\u001b[38;5;250;43m \u001b[39;49m\u001b[34;43;01mFakeSchemaPackable\u001b[39;49;00m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[43matdata\u001b[49m\u001b[43m.\u001b[49m\u001b[43mPackableSample\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mFakeSchema\u001b[49m\u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m 5\u001b[39m \u001b[43m \u001b[49m\u001b[43mblah\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mhello\u001b[39;49m\u001b[33;43m'\u001b[39;49m\n", 246 246 - "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.12.11-macos-aarch64-none/lib/python3.12/dataclasses.py:1265\u001b[39m, in \u001b[36mdataclass.<locals>.wrap\u001b[39m\u001b[34m(cls)\u001b[39m\n\u001b[32m 1264\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mwrap\u001b[39m(\u001b[38;5;28mcls\u001b[39m):\n\u001b[32m-> \u001b[39m\u001b[32m1265\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_process_class\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minit\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mrepr\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morder\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43munsafe_hash\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1266\u001b[39m \u001b[43m \u001b[49m\u001b[43mfrozen\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmatch_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkw_only\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mslots\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1267\u001b[39m \u001b[43m \u001b[49m\u001b[43mweakref_slot\u001b[49m\u001b[43m)\u001b[49m\n", 247 247 - "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.12.11-macos-aarch64-none/lib/python3.12/dataclasses.py:1024\u001b[39m, in \u001b[36m_process_class\u001b[39m\u001b[34m(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots, weakref_slot)\u001b[39m\n\u001b[32m 1021\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m has_dataclass_bases:\n\u001b[32m 1022\u001b[39m \u001b[38;5;66;03m# Raise an exception if any of our bases are frozen, but we're not.\u001b[39;00m\n\u001b[32m 1023\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m any_frozen_base \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m frozen:\n\u001b[32m-> \u001b[39m\u001b[32m1024\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[33m'\u001b[39m\u001b[33mcannot inherit non-frozen dataclass from a \u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 1025\u001b[39m \u001b[33m'\u001b[39m\u001b[33mfrozen one\u001b[39m\u001b[33m'\u001b[39m)\n\u001b[32m 1027\u001b[39m \u001b[38;5;66;03m# Raise an exception if we're frozen, but none of our bases are.\u001b[39;00m\n\u001b[32m 1028\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m any_frozen_base \u001b[38;5;129;01mand\u001b[39;00m frozen:\n", 248 248 - "\u001b[31mTypeError\u001b[39m: cannot inherit non-frozen dataclass from a frozen one" 249 249 - ] 250 250 - } 251 251 - ], 252 252 - "source": [ 253 253 - "from schemamodels.dynamic import FakeSchema\n", 254 254 - "\n", 255 255 - "@dataclass( frozen = False )\n", 256 256 - "class FakeSchemaPackable( atdata.PackableSample, FakeSchema ):\n", 257 257 - " blah = 'hello'" 258 258 - ] 259 259 - }, 260 260 - { 261 261 - "cell_type": "code", 262 262 - "execution_count": 21, 263 263 - "id": "3b6c01e8", 264 264 - "metadata": {}, 265 265 - "outputs": [], 266 266 - "source": [ 267 267 - "y = FakeSchema( property_a = 4, property_b = 'test' )" 268 268 - ] 269 269 - }, 270 270 - { 271 271 - "cell_type": "code", 272 272 - "execution_count": 26, 273 273 - "id": "6a1816d5", 274 274 - "metadata": {}, 275 275 - "outputs": [ 276 276 - { 277 277 - "data": { 278 278 - "text/plain": [ 279 279 - "4" 280 280 - ] 281 281 - }, 282 282 - "execution_count": 26, 283 283 - "metadata": {}, 284 284 - "output_type": "execute_result" 285 285 - } 286 286 - ], 287 287 - "source": [ 288 288 - "y.property_a" 289 289 - ] 290 290 - }, 291 291 - { 292 292 - "cell_type": "code", 293 293 - "execution_count": 28, 294 294 - "id": "64c6935f", 295 295 - "metadata": {}, 296 296 - "outputs": [], 297 297 - "source": [ 298 298 - "x = FakeSchemaPackable( property_a = 4, property_b = 'test' )" 299 299 - ] 300 300 - }, 301 301 - { 302 302 - "cell_type": "code", 303 303 - "execution_count": 17, 304 304 - "id": "0e5aaf10", 305 305 - "metadata": {}, 306 306 - "outputs": [ 307 307 - { 308 308 - "data": { 309 309 - "text/plain": [ 310 310 - "4" 311 311 - ] 312 312 - }, 313 313 - "execution_count": 17, 314 314 - "metadata": {}, 315 315 - "output_type": "execute_result" 316 316 - } 317 317 - ], 318 318 - "source": [ 319 319 - "x.property_a" 320 320 - ] 321 321 - }, 322 322 - { 323 323 - "cell_type": "code", 324 324 - "execution_count": 15, 325 325 - "id": "57cd50bc", 326 326 - "metadata": {}, 327 327 - "outputs": [ 328 328 - { 329 329 - "data": { 330 330 - "text/plain": [ 331 331 - "FakeSchemaPackable(property_b='', property_a=5)" 332 332 - ] 333 333 - }, 334 334 - "execution_count": 15, 335 335 - "metadata": {}, 336 336 - "output_type": "execute_result" 337 337 - } 338 338 - ], 339 339 - "source": [ 340 340 - "FakeSchemaPackable.from_bytes( x.as_wds['msgpack'] )" 341 341 - ] 342 342 - }, 343 343 - { 344 344 - "cell_type": "code", 345 345 - "execution_count": 18, 346 346 - "id": "0e97b846", 347 347 - "metadata": {}, 348 348 - "outputs": [], 349 349 - "source": [ 350 350 - "xw = x.as_wds" 351 351 - ] 352 352 - }, 353 353 - { 354 354 - "cell_type": "code", 355 355 - "execution_count": 19, 356 356 - "id": "ff625d70", 357 357 - "metadata": {}, 358 358 - "outputs": [ 359 359 - { 360 360 - "data": { 361 361 - "text/plain": [ 362 362 - "{'__key__': '36e2f3c0-cb2e-11f0-8000-000000000000', 'msgpack': b'\\x80'}" 363 363 - ] 364 364 - }, 365 365 - "execution_count": 19, 366 366 - "metadata": {}, 367 367 - "output_type": "execute_result" 368 368 - } 369 369 - ], 370 370 - "source": [ 371 371 - "xw" 372 372 - ] 373 373 - } 374 374 - ], 375 375 - "metadata": { 376 376 - "kernelspec": { 377 377 - "display_name": "atdata", 378 378 - "language": "python", 379 379 - "name": "python3" 380 380 - }, 381 381 - "language_info": { 382 382 - "codemirror_mode": { 383 383 - "name": "ipython", 384 384 - "version": 3 385 385 - }, 386 386 - "file_extension": ".py", 387 387 - "mimetype": "text/x-python", 388 388 - "name": "python", 389 389 - "nbconvert_exporter": "python", 390 390 - "pygments_lexer": "ipython3", 391 391 - "version": "3.12.11" 392 392 - } 393 393 - }, 394 394 - "nbformat": 4, 395 395 - "nbformat_minor": 5 396 396 - }