+1 -1

prototyping/.credentials/.gitignore

reviewed

··· 1 1 - *.env 1 1 + *.env

+193

prototyping/redis-noodling-2.ipynb

reviewed

··· 1 1 + { 2 2 + "cells": [ 3 3 + { 4 4 + "cell_type": "code", 5 5 + "execution_count": null, 6 6 + "id": "f2fd24f9", 7 7 + "metadata": {}, 8 8 + "outputs": [ 9 9 + { 10 10 + "ename": "ConnectionError", 11 11 + "evalue": "Error 61 connecting to localhost:6379. Connection refused.", 12 12 + "output_type": "error", 13 13 + "traceback": [ 14 14 + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", 15 15 + "\u001b[31mConnectionRefusedError\u001b[39m Traceback (most recent call last)", 16 16 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:378\u001b[39m, in \u001b[36mAbstractConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 377\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m378\u001b[39m sock = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mretry\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcall_with_retry\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 379\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_connect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mlambda\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mdisconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43merror\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 380\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 381\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.timeout:\n", 17 17 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/retry.py:62\u001b[39m, in \u001b[36mRetry.call_with_retry\u001b[39m\u001b[34m(self, do, fail)\u001b[39m\n\u001b[32m 61\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m62\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mdo\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 63\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;28mself\u001b[39m._supported_errors \u001b[38;5;28;01mas\u001b[39;00m error:\n", 18 18 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:379\u001b[39m, in \u001b[36mAbstractConnection.connect.<locals>.<lambda>\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 377\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 378\u001b[39m sock = \u001b[38;5;28mself\u001b[39m.retry.call_with_retry(\n\u001b[32m--> \u001b[39m\u001b[32m379\u001b[39m \u001b[38;5;28;01mlambda\u001b[39;00m: \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_connect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m, \u001b[38;5;28;01mlambda\u001b[39;00m error: \u001b[38;5;28mself\u001b[39m.disconnect(error)\n\u001b[32m 380\u001b[39m )\n\u001b[32m 381\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m socket.timeout:\n", 19 19 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:764\u001b[39m, in \u001b[36mConnection._connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 763\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m err \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m764\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[32m 765\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33msocket.getaddrinfo returned an empty list\u001b[39m\u001b[33m\"\u001b[39m)\n", 20 20 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:752\u001b[39m, in \u001b[36mConnection._connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 751\u001b[39m \u001b[38;5;66;03m# connect\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m752\u001b[39m \u001b[43msock\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msocket_address\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 754\u001b[39m \u001b[38;5;66;03m# set the socket_timeout now that we're connected\u001b[39;00m\n", 21 21 + "\u001b[31mConnectionRefusedError\u001b[39m: [Errno 61] Connection refused", 22 22 + "\nDuring handling of the above exception, another exception occurred:\n", 23 23 + "\u001b[31mConnectionError\u001b[39m Traceback (most recent call last)", 24 24 + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 4\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01matdata\u001b[39;00m\n\u001b[32m 2\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01matdata\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mlocal\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mal\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m4\u001b[39m local_index = \u001b[43mal\u001b[49m\u001b[43m.\u001b[49m\u001b[43mIndex\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 5\u001b[39m x: al.IndexEntry = local_index.list()[\u001b[32m0\u001b[39m]\n", 25 25 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/src/atdata/local.py:61\u001b[39m, in \u001b[36mIndex.__init__\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 55\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"TODO\"\"\"\u001b[39;00m\n\u001b[32m 56\u001b[39m \u001b[38;5;66;03m##\u001b[39;00m\n\u001b[32m 57\u001b[39m \n\u001b[32m 58\u001b[39m \u001b[38;5;66;03m# ...\u001b[39;00m\n\u001b[32m 59\u001b[39m \n\u001b[32m 60\u001b[39m \u001b[38;5;66;03m# Needed before we can do anything with redis-om queries\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m61\u001b[39m \u001b[43mMigrator\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", 26 26 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis_om/model/migrations/migrator.py:163\u001b[39m, in \u001b[36mMigrator.run\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 160\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mrun\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[32m 161\u001b[39m \u001b[38;5;66;03m# TODO: Migration history\u001b[39;00m\n\u001b[32m 162\u001b[39m \u001b[38;5;66;03m# TODO: Dry run with output\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m163\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mdetect_migrations\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 164\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m migration \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m.migrations:\n\u001b[32m 165\u001b[39m migration.run()\n", 27 27 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis_om/model/migrations/migrator.py:118\u001b[39m, in \u001b[36mMigrator.detect_migrations\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 115\u001b[39m current_hash = hashlib.sha1(schema.encode(\u001b[33m\"\u001b[39m\u001b[33mutf-8\u001b[39m\u001b[33m\"\u001b[39m)).hexdigest() \u001b[38;5;66;03m# nosec\u001b[39;00m\n\u001b[32m 117\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m118\u001b[39m \u001b[43mconn\u001b[49m\u001b[43m.\u001b[49m\u001b[43mft\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mMeta\u001b[49m\u001b[43m.\u001b[49m\u001b[43mindex_name\u001b[49m\u001b[43m)\u001b[49m\u001b[43m.\u001b[49m\u001b[43minfo\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 119\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m redis.ResponseError:\n\u001b[32m 120\u001b[39m \u001b[38;5;28mself\u001b[39m.migrations.append(\n\u001b[32m 121\u001b[39m IndexMigration(\n\u001b[32m 122\u001b[39m name,\n\u001b[32m (...)\u001b[39m\u001b[32m 128\u001b[39m )\n\u001b[32m 129\u001b[39m )\n", 28 28 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/commands/search/commands.py:452\u001b[39m, in \u001b[36mSearchCommands.info\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 444\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34minfo\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[32m 445\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"\u001b[39;00m\n\u001b[32m 446\u001b[39m \u001b[33;03m Get info an stats about the the current index, including the number of\u001b[39;00m\n\u001b[32m 447\u001b[39m \u001b[33;03m documents, memory consumption, etc\u001b[39;00m\n\u001b[32m 448\u001b[39m \n\u001b[32m 449\u001b[39m \u001b[33;03m For more information see `FT.INFO <https://redis.io/commands/ft.info>`_.\u001b[39;00m\n\u001b[32m 450\u001b[39m \u001b[33;03m \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m452\u001b[39m res = \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mexecute_command\u001b[49m\u001b[43m(\u001b[49m\u001b[43mINFO_CMD\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43mindex_name\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 453\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m._parse_results(INFO_CMD, res)\n", 29 29 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/client.py:605\u001b[39m, in \u001b[36mRedis.execute_command\u001b[39m\u001b[34m(self, *args, **options)\u001b[39m\n\u001b[32m 604\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mexecute_command\u001b[39m(\u001b[38;5;28mself\u001b[39m, *args, **options):\n\u001b[32m--> \u001b[39m\u001b[32m605\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_execute_command\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43moptions\u001b[49m\u001b[43m)\u001b[49m\n", 30 30 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/client.py:611\u001b[39m, in \u001b[36mRedis._execute_command\u001b[39m\u001b[34m(self, *args, **options)\u001b[39m\n\u001b[32m 609\u001b[39m pool = \u001b[38;5;28mself\u001b[39m.connection_pool\n\u001b[32m 610\u001b[39m command_name = args[\u001b[32m0\u001b[39m]\n\u001b[32m--> \u001b[39m\u001b[32m611\u001b[39m conn = \u001b[38;5;28mself\u001b[39m.connection \u001b[38;5;129;01mor\u001b[39;00m \u001b[43mpool\u001b[49m\u001b[43m.\u001b[49m\u001b[43mget_connection\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 613\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m._single_connection_client:\n\u001b[32m 614\u001b[39m \u001b[38;5;28mself\u001b[39m.single_connection_lock.acquire()\n", 31 31 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/utils.py:183\u001b[39m, in \u001b[36mdeprecated_args.<locals>.decorator.<locals>.wrapper\u001b[39m\u001b[34m(*args, **kwargs)\u001b[39m\n\u001b[32m 178\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m arg \u001b[38;5;129;01min\u001b[39;00m provided_args:\n\u001b[32m 179\u001b[39m warn_deprecated_arg_usage(\n\u001b[32m 180\u001b[39m arg, func.\u001b[34m__name__\u001b[39m, reason, version, stacklevel=\u001b[32m3\u001b[39m\n\u001b[32m 181\u001b[39m )\n\u001b[32m--> \u001b[39m\u001b[32m183\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[43m*\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m*\u001b[49m\u001b[43m*\u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", 32 32 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:1483\u001b[39m, in \u001b[36mConnectionPool.get_connection\u001b[39m\u001b[34m(self, command_name, *keys, **options)\u001b[39m\n\u001b[32m 1479\u001b[39m \u001b[38;5;28mself\u001b[39m._in_use_connections.add(connection)\n\u001b[32m 1481\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m 1482\u001b[39m \u001b[38;5;66;03m# ensure this connection is connected to Redis\u001b[39;00m\n\u001b[32m-> \u001b[39m\u001b[32m1483\u001b[39m \u001b[43mconnection\u001b[49m\u001b[43m.\u001b[49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 1484\u001b[39m \u001b[38;5;66;03m# connections that the pool provides should be ready to send\u001b[39;00m\n\u001b[32m 1485\u001b[39m \u001b[38;5;66;03m# a command. if not, the connection was either returned to the\u001b[39;00m\n\u001b[32m 1486\u001b[39m \u001b[38;5;66;03m# pool before all data has been read or the socket has been\u001b[39;00m\n\u001b[32m 1487\u001b[39m \u001b[38;5;66;03m# closed. either way, reconnect and verify everything is good.\u001b[39;00m\n\u001b[32m 1488\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n", 33 33 + "\u001b[36mFile \u001b[39m\u001b[32m~/git-forecast/atdata/.venv/lib/python3.12/site-packages/redis/connection.py:384\u001b[39m, in \u001b[36mAbstractConnection.connect\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 382\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTimeoutError\u001b[39;00m(\u001b[33m\"\u001b[39m\u001b[33mTimeout connecting to server\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 383\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[32m--> \u001b[39m\u001b[32m384\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(\u001b[38;5;28mself\u001b[39m._error_message(e))\n\u001b[32m 386\u001b[39m \u001b[38;5;28mself\u001b[39m._sock = sock\n\u001b[32m 387\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n", 34 34 + "\u001b[31mConnectionError\u001b[39m: Error 61 connecting to localhost:6379. Connection refused." 35 35 + ] 36 36 + } 37 37 + ], 38 38 + "source": [ 39 39 + "import atdata\n", 40 40 + "import atdata._local_dev as al\n", 41 41 + "\n", 42 42 + "local_index = al.Index()\n", 43 43 + "x: al.IndexEntry = local_index.list()[0]" 44 44 + ] 45 45 + }, 46 46 + { 47 47 + "cell_type": "code", 48 48 + "execution_count": 4, 49 49 + "id": "c968a5c7", 50 50 + "metadata": {}, 51 51 + "outputs": [ 52 52 + { 53 53 + "data": { 54 54 + "text/plain": [ 55 55 + "{'$id': 'https://schema.dev/fake-schema.schema.json',\n", 56 56 + " '$schema': 'http://json-schema.org/draft-07/schema#',\n", 57 57 + " 'title': 'fake-schema',\n", 58 58 + " 'description': 'Blue Blah',\n", 59 59 + " 'type': 'object',\n", 60 60 + " 'properties': {'property_a': {'default': 5, 'type': 'integer'},\n", 61 61 + " 'property_b': {'type': 'string'}}}" 62 62 + ] 63 63 + }, 64 64 + "execution_count": 4, 65 65 + "metadata": {}, 66 66 + "output_type": "execute_result" 67 67 + } 68 68 + ], 69 69 + "source": [ 70 70 + "x.sample_schema.json_schema" 71 71 + ] 72 72 + }, 73 73 + { 74 74 + "cell_type": "code", 75 75 + "execution_count": 6, 76 76 + "id": "d4761aa8", 77 77 + "metadata": {}, 78 78 + "outputs": [], 79 79 + "source": [ 80 80 + "from pydantic import create_model, Field" 81 81 + ] 82 82 + }, 83 83 + { 84 84 + "cell_type": "code", 85 85 + "execution_count": 7, 86 86 + "id": "3282fd1d", 87 87 + "metadata": {}, 88 88 + "outputs": [], 89 89 + "source": [ 90 90 + "json_schema = x.sample_schema.json_schema\n", 91 91 + "\n", 92 92 + "# Map JSON Schema types and constraints to Pydantic fields\n", 93 93 + "fields = {}\n", 94 94 + "for prop_name, prop_details in json_schema[\"properties\"].items():\n", 95 95 + " field_type = None\n", 96 96 + " if prop_details[\"type\"] == \"string\":\n", 97 97 + " if prop_details.get(\"format\") == \"email\":\n", 98 98 + " field_type = str # Pydantic's EmailStr can be used for stronger validation\n", 99 99 + " else:\n", 100 100 + " field_type = str\n", 101 101 + " elif prop_details[\"type\"] == \"integer\":\n", 102 102 + " field_type = int\n", 103 103 + " # Add more type mappings as needed\n", 104 104 + "\n", 105 105 + " field_args = {}\n", 106 106 + " if \"maxLength\" in prop_details:\n", 107 107 + " field_args[\"max_length\"] = prop_details[\"maxLength\"]\n", 108 108 + " if \"minimum\" in prop_details:\n", 109 109 + " field_args[\"ge\"] = prop_details[\"minimum\"]\n", 110 110 + " # Add more constraint mappings (e.g., pattern, minLength, maximum, etc.)\n", 111 111 + "\n", 112 112 + " if prop_name in json_schema.get(\"required\", []):\n", 113 113 + " fields[prop_name] = (field_type, Field(**field_args))\n", 114 114 + " else:\n", 115 115 + " fields[prop_name] = (field_type, Field(None, **field_args)) # Optional field" 116 116 + ] 117 117 + }, 118 118 + { 119 119 + "cell_type": "code", 120 120 + "execution_count": 27, 121 121 + "id": "f9dc5833", 122 122 + "metadata": {}, 123 123 + "outputs": [], 124 124 + "source": [ 125 125 + "_FakeSchema = create_model( 'FakeSchema', **fields )\n", 126 126 + "class FakeSchema( _FakeSchema, atdata.PackableSample ):\n", 127 127 + " pass" 128 128 + ] 129 129 + }, 130 130 + { 131 131 + "cell_type": "code", 132 132 + "execution_count": 37, 133 133 + "id": "41ed6eee", 134 134 + "metadata": {}, 135 135 + "outputs": [], 136 136 + "source": [ 137 137 + "x = FakeSchema( property_a = 2, property_b = 'hello' )" 138 138 + ] 139 139 + }, 140 140 + { 141 141 + "cell_type": "code", 142 142 + "execution_count": 38, 143 143 + "id": "1f6f3f97", 144 144 + "metadata": {}, 145 145 + "outputs": [], 146 146 + "source": [ 147 147 + "x_prime = FakeSchema.from_bytes( x.as_wds['msgpack'] )" 148 148 + ] 149 149 + }, 150 150 + { 151 151 + "cell_type": "code", 152 152 + "execution_count": 39, 153 153 + "id": "bd139ab0", 154 154 + "metadata": {}, 155 155 + "outputs": [ 156 156 + { 157 157 + "data": { 158 158 + "text/plain": [ 159 159 + "FakeSchema(property_a=2, property_b='hello')" 160 160 + ] 161 161 + }, 162 162 + "execution_count": 39, 163 163 + "metadata": {}, 164 164 + "output_type": "execute_result" 165 165 + } 166 166 + ], 167 167 + "source": [ 168 168 + "x_prime" 169 169 + ] 170 170 + } 171 171 + ], 172 172 + "metadata": { 173 173 + "kernelspec": { 174 174 + "display_name": "atdata", 175 175 + "language": "python", 176 176 + "name": "python3" 177 177 + }, 178 178 + "language_info": { 179 179 + "codemirror_mode": { 180 180 + "name": "ipython", 181 181 + "version": 3 182 182 + }, 183 183 + "file_extension": ".py", 184 184 + "mimetype": "text/x-python", 185 185 + "name": "python", 186 186 + "nbconvert_exporter": "python", 187 187 + "pygments_lexer": "ipython3", 188 188 + "version": "3.12.11" 189 189 + } 190 190 + }, 191 191 + "nbformat": 4, 192 192 + "nbformat_minor": 5 193 193 + }

+132

prototyping/redis-noodling-3.ipynb

reviewed

··· 1 1 + { 2 2 + "cells": [ 3 3 + { 4 4 + "cell_type": "code", 5 5 + "execution_count": 1, 6 6 + "id": "256e5c04", 7 7 + "metadata": {}, 8 8 + "outputs": [], 9 9 + "source": [ 10 10 + "import atdata\n", 11 11 + "import atdata.local as al" 12 12 + ] 13 13 + }, 14 14 + { 15 15 + "cell_type": "code", 16 16 + "execution_count": 5, 17 17 + "id": "d9df3d99", 18 18 + "metadata": {}, 19 19 + "outputs": [], 20 20 + "source": [ 21 21 + "from redis import Redis\n", 22 22 + "\n", 23 23 + "test_url = \"rediss://default:AWWlAAIncDEzZGM2ZGM2NjJiMWE0N2EwYjI0ZWM1ZTcxYWIyYjIxZXAxMjYwMjE@settled-moose-26021.upstash.io:6379\"\n", 24 24 + "r = Redis.from_url( test_url )" 25 25 + ] 26 26 + }, 27 27 + { 28 28 + "cell_type": "code", 29 29 + "execution_count": 6, 30 30 + "id": "a98b9f90", 31 31 + "metadata": {}, 32 32 + "outputs": [ 33 33 + { 34 34 + "data": { 35 35 + "text/plain": [ 36 36 + "True" 37 37 + ] 38 38 + }, 39 39 + "execution_count": 6, 40 40 + "metadata": {}, 41 41 + "output_type": "execute_result" 42 42 + } 43 43 + ], 44 44 + "source": [ 45 45 + "r.set( 'baz', 'bar' )" 46 46 + ] 47 47 + }, 48 48 + { 49 49 + "cell_type": "code", 50 50 + "execution_count": 8, 51 51 + "id": "47b08315", 52 52 + "metadata": {}, 53 53 + "outputs": [ 54 54 + { 55 55 + "data": { 56 56 + "text/plain": [ 57 57 + "b'bar'" 58 58 + ] 59 59 + }, 60 60 + "execution_count": 8, 61 61 + "metadata": {}, 62 62 + "output_type": "execute_result" 63 63 + } 64 64 + ], 65 65 + "source": [ 66 66 + "r.get( 'baz' )" 67 67 + ] 68 68 + }, 69 69 + { 70 70 + "cell_type": "code", 71 71 + "execution_count": 9, 72 72 + "id": "b1a1e2f2", 73 73 + "metadata": {}, 74 74 + "outputs": [], 75 75 + "source": [ 76 76 + "redis_index = al.Index( redis = r )" 77 77 + ] 78 78 + }, 79 79 + { 80 80 + "cell_type": "code", 81 81 + "execution_count": 10, 82 82 + "id": "10aab6ce", 83 83 + "metadata": {}, 84 84 + "outputs": [ 85 85 + { 86 86 + "data": { 87 87 + "text/plain": [ 88 88 + "[]" 89 89 + ] 90 90 + }, 91 91 + "execution_count": 10, 92 92 + "metadata": {}, 93 93 + "output_type": "execute_result" 94 94 + } 95 95 + ], 96 96 + "source": [ 97 97 + "redis_index.list()" 98 98 + ] 99 99 + }, 100 100 + { 101 101 + "cell_type": "code", 102 102 + "execution_count": null, 103 103 + "id": "b5f66fda", 104 104 + "metadata": {}, 105 105 + "outputs": [], 106 106 + "source": [ 107 107 + "ds = atdata.Dataset( )" 108 108 + ] 109 109 + } 110 110 + ], 111 111 + "metadata": { 112 112 + "kernelspec": { 113 113 + "display_name": "atdata", 114 114 + "language": "python", 115 115 + "name": "python3" 116 116 + }, 117 117 + "language_info": { 118 118 + "codemirror_mode": { 119 119 + "name": "ipython", 120 120 + "version": 3 121 121 + }, 122 122 + "file_extension": ".py", 123 123 + "mimetype": "text/x-python", 124 124 + "name": "python", 125 125 + "nbconvert_exporter": "python", 126 126 + "pygments_lexer": "ipython3", 127 127 + "version": "3.12.11" 128 128 + } 129 129 + }, 130 130 + "nbformat": 4, 131 131 + "nbformat_minor": 5 132 132 + }

+203

prototyping/redis-noodling-4.ipynb

reviewed

··· 1 1 + { 2 2 + "cells": [ 3 3 + { 4 4 + "cell_type": "code", 5 5 + "execution_count": 1, 6 6 + "id": "bcb96018", 7 7 + "metadata": {}, 8 8 + "outputs": [], 9 9 + "source": [ 10 10 + "from redis import Redis\n", 11 11 + "\n", 12 12 + "import atdata.local as al" 13 13 + ] 14 14 + }, 15 15 + { 16 16 + "cell_type": "code", 17 17 + "execution_count": 2, 18 18 + "id": "d1631435", 19 19 + "metadata": {}, 20 20 + "outputs": [], 21 21 + "source": [ 22 22 + "test_url = \"rediss://default:AWWlAAIncDEzZGM2ZGM2NjJiMWE0N2EwYjI0ZWM1ZTcxYWIyYjIxZXAxMjYwMjE@settled-moose-26021.upstash.io:6379\"\n", 23 23 + "test_redis = Redis.from_url( test_url )" 24 24 + ] 25 25 + }, 26 26 + { 27 27 + "cell_type": "code", 28 28 + "execution_count": 3, 29 29 + "id": "5c9ee2e2", 30 30 + "metadata": {}, 31 31 + "outputs": [], 32 32 + "source": [ 33 33 + "local_repo = al.Repo(\n", 34 34 + " s3_credentials = '.credentials/r2-analysis-hive.env',\n", 35 35 + " hive_path = 'analysis-hive/test-hive/',\n", 36 36 + " redis = test_redis,\n", 37 37 + ")" 38 38 + ] 39 39 + }, 40 40 + { 41 41 + "cell_type": "code", 42 42 + "execution_count": 4, 43 43 + "id": "4425dc0d", 44 44 + "metadata": {}, 45 45 + "outputs": [], 46 46 + "source": [ 47 47 + "import atdata\n", 48 48 + "import webdataset as wds\n", 49 49 + "from dataclasses import dataclass\n", 50 50 + "from numpy.typing import NDArray\n", 51 51 + "\n", 52 52 + "@dataclass\n", 53 53 + "class TestSample( atdata.PackableSample ):\n", 54 54 + " identifier: int\n", 55 55 + " values: NDArray" 56 56 + ] 57 57 + }, 58 58 + { 59 59 + "cell_type": "code", 60 60 + "execution_count": 5, 61 61 + "id": "8c2ed4b2", 62 62 + "metadata": {}, 63 63 + "outputs": [ 64 64 + { 65 65 + "name": "stdout", 66 66 + "output_type": "stream", 67 67 + "text": [ 68 68 + "# writing data/test-000000.tar 0 0.0 GB 0\n", 69 69 + "# writing data/test-000001.tar 100 0.2 GB 100\n", 70 70 + "# writing data/test-000002.tar 100 0.2 GB 200\n", 71 71 + "# writing data/test-000003.tar 100 0.2 GB 300\n", 72 72 + "# writing data/test-000004.tar 100 0.2 GB 400\n", 73 73 + "# writing data/test-000005.tar 100 0.2 GB 500\n", 74 74 + "# writing data/test-000006.tar 100 0.2 GB 600\n", 75 75 + "# writing data/test-000007.tar 100 0.2 GB 700\n", 76 76 + "# writing data/test-000008.tar 100 0.2 GB 800\n", 77 77 + "# writing data/test-000009.tar 100 0.2 GB 900\n" 78 78 + ] 79 79 + } 80 80 + ], 81 81 + "source": [ 82 82 + "import numpy as np\n", 83 83 + "n = 950\n", 84 84 + "\n", 85 85 + "import os\n", 86 86 + "os.makedirs( 'data', exist_ok = True )\n", 87 87 + "\n", 88 88 + "with wds.writer.ShardWriter( 'data/test-%06d.tar', maxcount = 100 ) as sink:\n", 89 89 + " for i in range( n ):\n", 90 90 + " new_sample = TestSample(\n", 91 91 + " identifier = i + 1,\n", 92 92 + " values = np.random.normal( size = (1024, 256) ),\n", 93 93 + " )\n", 94 94 + " sink.write( new_sample.as_wds )" 95 95 + ] 96 96 + }, 97 97 + { 98 98 + "cell_type": "code", 99 99 + "execution_count": 6, 100 100 + "id": "ba794054", 101 101 + "metadata": {}, 102 102 + "outputs": [ 103 103 + { 104 104 + "name": "stdout", 105 105 + "output_type": "stream", 106 106 + "text": [ 107 107 + "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000000.tar 0 0.0 GB 0\n", 108 108 + "Copying file to s3 ... done.\n", 109 109 + "Deleting local cache file ... done.\n", 110 110 + "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000001.tar 150 0.3 GB 150\n", 111 111 + "Copying file to s3 ... done.\n", 112 112 + "Deleting local cache file ... done.\n", 113 113 + "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000002.tar 150 0.3 GB 300\n", 114 114 + "Copying file to s3 ... done.\n", 115 115 + "Deleting local cache file ... done.\n", 116 116 + "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000003.tar 150 0.3 GB 450\n", 117 117 + "Copying file to s3 ... done.\n", 118 118 + "Deleting local cache file ... done.\n", 119 119 + "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000004.tar 150 0.3 GB 600\n", 120 120 + "Copying file to s3 ... done.\n", 121 121 + "Deleting local cache file ... done.\n", 122 122 + "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000005.tar 150 0.3 GB 750\n", 123 123 + "Copying file to s3 ... done.\n", 124 124 + "Deleting local cache file ... done.\n", 125 125 + "# writing analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--000006.tar 150 0.3 GB 900\n", 126 126 + "Copying file to s3 ... done.\n", 127 127 + "Deleting local cache file ... done.\n" 128 128 + ] 129 129 + }, 130 130 + { 131 131 + "data": { 132 132 + "text/plain": [ 133 133 + "(BasicIndexEntry(wds_url='analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--{000000..-00001}.tar', sample_kind='__main__.TestSample', metadata_url='analysis-hive/test-hive/metadata/atdata-metadata--bf546505-3e70-44d4-ab1f-9729ad25ce8c.msgpack', uuid='bf546505-3e70-44d4-ab1f-9729ad25ce8c'),\n", 134 134 + " <atdata.dataset.Dataset at 0x111daa9f0>)" 135 135 + ] 136 136 + }, 137 137 + "execution_count": 6, 138 138 + "metadata": {}, 139 139 + "output_type": "execute_result" 140 140 + } 141 141 + ], 142 142 + "source": [ 143 143 + "new_dataset = atdata.Dataset[TestSample]( 'data/test-{000000..000009}.tar' )\n", 144 144 + "local_repo.insert( new_dataset,\n", 145 145 + " maxcount = 150,\n", 146 146 + " cache_local = True,\n", 147 147 + ")" 148 148 + ] 149 149 + }, 150 150 + { 151 151 + "cell_type": "code", 152 152 + "execution_count": 8, 153 153 + "id": "31f5f666", 154 154 + "metadata": {}, 155 155 + "outputs": [], 156 156 + "source": [ 157 157 + "x = local_repo.index.list()[0]" 158 158 + ] 159 159 + }, 160 160 + { 161 161 + "cell_type": "code", 162 162 + "execution_count": 7, 163 163 + "id": "85924ef6", 164 164 + "metadata": {}, 165 165 + "outputs": [ 166 166 + { 167 167 + "name": "stdout", 168 168 + "output_type": "stream", 169 169 + "text": [ 170 170 + "BasicIndexEntry(wds_url='analysis-hive/test-hive/atdata--138f45b5-3ba4-46b7-9521-4930093e2ffc--{000000..000000}.tar', sample_kind='atdata.local.T', metadata_url='analysis-hive/test-hive/metadata/atdata-metadata--138f45b5-3ba4-46b7-9521-4930093e2ffc.msgpack', uuid='138f45b5-3ba4-46b7-9521-4930093e2ffc')\n", 171 171 + "BasicIndexEntry(wds_url='analysis-hive/test-hive/atdata--66854cd0-1c3e-4ad2-a757-e100d566a878--{000000..000006}.tar', sample_kind='__main__.TestSample', metadata_url='analysis-hive/test-hive/metadata/atdata-metadata--66854cd0-1c3e-4ad2-a757-e100d566a878.msgpack', uuid='66854cd0-1c3e-4ad2-a757-e100d566a878')\n", 172 172 + "BasicIndexEntry(wds_url='analysis-hive/test-hive/atdata--bf546505-3e70-44d4-ab1f-9729ad25ce8c--{000000..-00001}.tar', sample_kind='__main__.TestSample', metadata_url='analysis-hive/test-hive/metadata/atdata-metadata--bf546505-3e70-44d4-ab1f-9729ad25ce8c.msgpack', uuid='bf546505-3e70-44d4-ab1f-9729ad25ce8c')\n" 173 173 + ] 174 174 + } 175 175 + ], 176 176 + "source": [ 177 177 + "for e in local_repo.index.entries:\n", 178 178 + " print( e )" 179 179 + ] 180 180 + } 181 181 + ], 182 182 + "metadata": { 183 183 + "kernelspec": { 184 184 + "display_name": "atdata", 185 185 + "language": "python", 186 186 + "name": "python3" 187 187 + }, 188 188 + "language_info": { 189 189 + "codemirror_mode": { 190 190 + "name": "ipython", 191 191 + "version": 3 192 192 + }, 193 193 + "file_extension": ".py", 194 194 + "mimetype": "text/x-python", 195 195 + "name": "python", 196 196 + "nbconvert_exporter": "python", 197 197 + "pygments_lexer": "ipython3", 198 198 + "version": "3.12.11" 199 199 + } 200 200 + }, 201 201 + "nbformat": 4, 202 202 + "nbformat_minor": 5 203 203 + }

+396

prototyping/redis-noodling.ipynb

reviewed

··· 1 1 + { 2 2 + "cells": [ 3 3 + { 4 4 + "cell_type": "code", 5 5 + "execution_count": null, 6 6 + "id": "177dfb94", 7 7 + "metadata": {}, 8 8 + "outputs": [], 9 9 + "source": [ 10 10 + "import atdata\n", 11 11 + "from dataclasses import dataclass\n", 12 12 + "import atdata._local_dev as al" 13 13 + ] 14 14 + }, 15 15 + { 16 16 + "cell_type": "code", 17 17 + "execution_count": 2, 18 18 + "id": "f9cb805d", 19 19 + "metadata": {}, 20 20 + "outputs": [], 21 21 + "source": [ 22 22 + "import redis" 23 23 + ] 24 24 + }, 25 25 + { 26 26 + "cell_type": "code", 27 27 + "execution_count": 3, 28 28 + "id": "48e3c969", 29 29 + "metadata": {}, 30 30 + "outputs": [ 31 31 + { 32 32 + "data": { 33 33 + "text/plain": [ 34 34 + "[b':atdata.local.IndexEntry:01KB1CEARQXTTZDP1A8211645V',\n", 35 35 + " b':atdata.local.IndexEntry:01KB1D7R2SG9EDX5EHEFEC8R2N',\n", 36 36 + " b':atdata.local.IndexEntry:01KB1DDAW9HQ99XFF8FPFQ5AVT',\n", 37 37 + " b':atdata.local.IndexEntry:01KB1DMKTFGHY0ZH3VTNJ1H9XG',\n", 38 38 + " b':atdata.local.IndexEntry:index:hash',\n", 39 39 + " b':atdata.local.IndexEntry:01KB1CEHX24E9VDGG23DSMDZAF']" 40 40 + ] 41 41 + }, 42 42 + "execution_count": 3, 43 43 + "metadata": {}, 44 44 + "output_type": "execute_result" 45 45 + } 46 46 + ], 47 47 + "source": [ 48 48 + "pool = redis.ConnectionPool(host='localhost', port=6379, db=0)\n", 49 49 + "r = redis.Redis(connection_pool=pool)\n", 50 50 + "\n", 51 51 + "# Check specific keys\n", 52 52 + "r.keys('*')" 53 53 + ] 54 54 + }, 55 55 + { 56 56 + "cell_type": "code", 57 57 + "execution_count": 4, 58 58 + "id": "69d8b045", 59 59 + "metadata": {}, 60 60 + "outputs": [], 61 61 + "source": [ 62 62 + "local_index = al.Index()" 63 63 + ] 64 64 + }, 65 65 + { 66 66 + "cell_type": "code", 67 67 + "execution_count": 5, 68 68 + "id": "b157c142", 69 69 + "metadata": {}, 70 70 + "outputs": [], 71 71 + "source": [ 72 72 + "@dataclass\n", 73 73 + "class TestSample( atdata.PackableSample ):\n", 74 74 + " identifier: int\n", 75 75 + " name: str\n", 76 76 + "\n", 77 77 + "dataset = atdata.Dataset[TestSample]( '/data/test.tar' )" 78 78 + ] 79 79 + }, 80 80 + { 81 81 + "cell_type": "code", 82 82 + "execution_count": 6, 83 83 + "id": "6cc845f4", 84 84 + "metadata": {}, 85 85 + "outputs": [ 86 86 + { 87 87 + "data": { 88 88 + "text/plain": [ 89 89 + "IndexEntry(pk='01KB1DTP0QJZGV49J1339T9256', wds_url='/data/test.tar', sample_schema=SampleSchema(pk='01KB1DTP0PQ4XKV2N5HHQC3GWK', identifier='74deedc1-fbbb-4b11-9f99-cd73d3257983', json_schema={'$id': 'https://schema.dev/fake-schema.schema.json', '$schema': 'http://json-schema.org/draft-07/schema#', 'title': 'fake-schema', 'description': 'Blue Blah', 'type': 'object', 'properties': {'property_a': {'default': 5, 'type': 'integer'}, 'property_b': {'type': 'string'}}}), metadata_schema=None)" 90 90 + ] 91 91 + }, 92 92 + "execution_count": 6, 93 93 + "metadata": {}, 94 94 + "output_type": "execute_result" 95 95 + } 96 96 + ], 97 97 + "source": [ 98 98 + "local_index.add( dataset )" 99 99 + ] 100 100 + }, 101 101 + { 102 102 + "cell_type": "code", 103 103 + "execution_count": 7, 104 104 + "id": "ae382e86", 105 105 + "metadata": {}, 106 106 + "outputs": [], 107 107 + "source": [ 108 108 + "x: al.IndexEntry = local_index.list()[0]" 109 109 + ] 110 110 + }, 111 111 + { 112 112 + "cell_type": "code", 113 113 + "execution_count": 8, 114 114 + "id": "7edf8718", 115 115 + "metadata": {}, 116 116 + "outputs": [ 117 117 + { 118 118 + "data": { 119 119 + "text/plain": [ 120 120 + "{'$id': 'https://schema.dev/fake-schema.schema.json',\n", 121 121 + " '$schema': 'http://json-schema.org/draft-07/schema#',\n", 122 122 + " 'title': 'fake-schema',\n", 123 123 + " 'description': 'Blue Blah',\n", 124 124 + " 'type': 'object',\n", 125 125 + " 'properties': {'property_a': {'default': 5, 'type': 'integer'},\n", 126 126 + " 'property_b': {'type': 'string'}}}" 127 127 + ] 128 128 + }, 129 129 + "execution_count": 8, 130 130 + "metadata": {}, 131 131 + "output_type": "execute_result" 132 132 + } 133 133 + ], 134 134 + "source": [ 135 135 + "x.sample_schema.json_schema" 136 136 + ] 137 137 + }, 138 138 + { 139 139 + "cell_type": "code", 140 140 + "execution_count": 46, 141 141 + "id": "618ec152", 142 142 + "metadata": {}, 143 143 + "outputs": [], 144 144 + "source": [ 145 145 + "def _test_init( self, a: int, b: str ):\n", 146 146 + " self.property_a = a\n", 147 147 + " self.property_b = b\n", 148 148 + "\n", 149 149 + "TestType = type( 'TestType', (object,), {\n", 150 150 + " '__init__': _test_init,\n", 151 151 + " 'property_a': 4,\n", 152 152 + " '__annotations__': {\n", 153 153 + " 'property_a': int,\n", 154 154 + " 'property_b': str,\n", 155 155 + " }\n", 156 156 + "} )" 157 157 + ] 158 158 + }, 159 159 + { 160 160 + "cell_type": "code", 161 161 + "execution_count": null, 162 162 + "id": "d7f52957", 163 163 + "metadata": {}, 164 164 + "outputs": [], 165 165 + "source": [ 166 166 + "TestType.property_a" 167 167 + ] 168 168 + }, 169 169 + { 170 170 + "cell_type": "code", 171 171 + "execution_count": 49, 172 172 + "id": "5cba0e62", 173 173 + "metadata": {}, 174 174 + "outputs": [], 175 175 + "source": [ 176 176 + "x = TestType( 4, 'shark' )" 177 177 + ] 178 178 + }, 179 179 + { 180 180 + "cell_type": "code", 181 181 + "execution_count": 50, 182 182 + "id": "7209e369", 183 183 + "metadata": {}, 184 184 + "outputs": [ 185 185 + { 186 186 + "data": { 187 187 + "text/plain": [ 188 188 + "4" 189 189 + ] 190 190 + }, 191 191 + "execution_count": 50, 192 192 + "metadata": {}, 193 193 + "output_type": "execute_result" 194 194 + } 195 195 + ], 196 196 + "source": [ 197 197 + "x.property_a" 198 198 + ] 199 199 + }, 200 200 + { 201 201 + "cell_type": "code", 202 202 + "execution_count": null, 203 203 + "id": "3e88a7e1", 204 204 + "metadata": {}, 205 205 + "outputs": [], 206 206 + "source": [] 207 207 + }, 208 208 + { 209 209 + "cell_type": "code", 210 210 + "execution_count": 9, 211 211 + "id": "2156e6cd", 212 212 + "metadata": {}, 213 213 + "outputs": [ 214 214 + { 215 215 + "data": { 216 216 + "text/plain": [ 217 217 + "True" 218 218 + ] 219 219 + }, 220 220 + "execution_count": 9, 221 221 + "metadata": {}, 222 222 + "output_type": "execute_result" 223 223 + } 224 224 + ], 225 225 + "source": [ 226 226 + "from schemamodels import SchemaModelFactory\n", 227 227 + "\n", 228 228 + "factory = SchemaModelFactory()\n", 229 229 + "factory.register( x.sample_schema.json_schema )" 230 230 + ] 231 231 + }, 232 232 + { 233 233 + "cell_type": "code", 234 234 + "execution_count": 31, 235 235 + "id": "f26d2901", 236 236 + "metadata": {}, 237 237 + "outputs": [ 238 238 + { 239 239 + "ename": "TypeError", 240 240 + "evalue": "cannot inherit non-frozen dataclass from a frozen one", 241 241 + "output_type": "error", 242 242 + "traceback": [ 243 243 + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", 244 244 + "\u001b[31mTypeError\u001b[39m Traceback (most recent call last)", 245 245 + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[31]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mschemamodels\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mdynamic\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m FakeSchema\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m \u001b[38;5;129;43m@dataclass\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[43mfrozen\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 4\u001b[39m \u001b[38;5;28;43;01mclass\u001b[39;49;00m\u001b[38;5;250;43m \u001b[39;49m\u001b[34;43;01mFakeSchemaPackable\u001b[39;49;00m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[43matdata\u001b[49m\u001b[43m.\u001b[49m\u001b[43mPackableSample\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mFakeSchema\u001b[49m\u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[32m 5\u001b[39m \u001b[43m \u001b[49m\u001b[43mblah\u001b[49m\u001b[43m \u001b[49m\u001b[43m=\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mhello\u001b[39;49m\u001b[33;43m'\u001b[39;49m\n", 246 246 + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.12.11-macos-aarch64-none/lib/python3.12/dataclasses.py:1265\u001b[39m, in \u001b[36mdataclass.<locals>.wrap\u001b[39m\u001b[34m(cls)\u001b[39m\n\u001b[32m 1264\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mwrap\u001b[39m(\u001b[38;5;28mcls\u001b[39m):\n\u001b[32m-> \u001b[39m\u001b[32m1265\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_process_class\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minit\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mrepr\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43meq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morder\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43munsafe_hash\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1266\u001b[39m \u001b[43m \u001b[49m\u001b[43mfrozen\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmatch_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkw_only\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mslots\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 1267\u001b[39m \u001b[43m \u001b[49m\u001b[43mweakref_slot\u001b[49m\u001b[43m)\u001b[49m\n", 247 247 + "\u001b[36mFile \u001b[39m\u001b[32m~/.local/share/uv/python/cpython-3.12.11-macos-aarch64-none/lib/python3.12/dataclasses.py:1024\u001b[39m, in \u001b[36m_process_class\u001b[39m\u001b[34m(cls, init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, slots, weakref_slot)\u001b[39m\n\u001b[32m 1021\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m has_dataclass_bases:\n\u001b[32m 1022\u001b[39m \u001b[38;5;66;03m# Raise an exception if any of our bases are frozen, but we're not.\u001b[39;00m\n\u001b[32m 1023\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m any_frozen_base \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m frozen:\n\u001b[32m-> \u001b[39m\u001b[32m1024\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[33m'\u001b[39m\u001b[33mcannot inherit non-frozen dataclass from a \u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m 1025\u001b[39m \u001b[33m'\u001b[39m\u001b[33mfrozen one\u001b[39m\u001b[33m'\u001b[39m)\n\u001b[32m 1027\u001b[39m \u001b[38;5;66;03m# Raise an exception if we're frozen, but none of our bases are.\u001b[39;00m\n\u001b[32m 1028\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m any_frozen_base \u001b[38;5;129;01mand\u001b[39;00m frozen:\n", 248 248 + "\u001b[31mTypeError\u001b[39m: cannot inherit non-frozen dataclass from a frozen one" 249 249 + ] 250 250 + } 251 251 + ], 252 252 + "source": [ 253 253 + "from schemamodels.dynamic import FakeSchema\n", 254 254 + "\n", 255 255 + "@dataclass( frozen = False )\n", 256 256 + "class FakeSchemaPackable( atdata.PackableSample, FakeSchema ):\n", 257 257 + " blah = 'hello'" 258 258 + ] 259 259 + }, 260 260 + { 261 261 + "cell_type": "code", 262 262 + "execution_count": 21, 263 263 + "id": "3b6c01e8", 264 264 + "metadata": {}, 265 265 + "outputs": [], 266 266 + "source": [ 267 267 + "y = FakeSchema( property_a = 4, property_b = 'test' )" 268 268 + ] 269 269 + }, 270 270 + { 271 271 + "cell_type": "code", 272 272 + "execution_count": 26, 273 273 + "id": "6a1816d5", 274 274 + "metadata": {}, 275 275 + "outputs": [ 276 276 + { 277 277 + "data": { 278 278 + "text/plain": [ 279 279 + "4" 280 280 + ] 281 281 + }, 282 282 + "execution_count": 26, 283 283 + "metadata": {}, 284 284 + "output_type": "execute_result" 285 285 + } 286 286 + ], 287 287 + "source": [ 288 288 + "y.property_a" 289 289 + ] 290 290 + }, 291 291 + { 292 292 + "cell_type": "code", 293 293 + "execution_count": 28, 294 294 + "id": "64c6935f", 295 295 + "metadata": {}, 296 296 + "outputs": [], 297 297 + "source": [ 298 298 + "x = FakeSchemaPackable( property_a = 4, property_b = 'test' )" 299 299 + ] 300 300 + }, 301 301 + { 302 302 + "cell_type": "code", 303 303 + "execution_count": 17, 304 304 + "id": "0e5aaf10", 305 305 + "metadata": {}, 306 306 + "outputs": [ 307 307 + { 308 308 + "data": { 309 309 + "text/plain": [ 310 310 + "4" 311 311 + ] 312 312 + }, 313 313 + "execution_count": 17, 314 314 + "metadata": {}, 315 315 + "output_type": "execute_result" 316 316 + } 317 317 + ], 318 318 + "source": [ 319 319 + "x.property_a" 320 320 + ] 321 321 + }, 322 322 + { 323 323 + "cell_type": "code", 324 324 + "execution_count": 15, 325 325 + "id": "57cd50bc", 326 326 + "metadata": {}, 327 327 + "outputs": [ 328 328 + { 329 329 + "data": { 330 330 + "text/plain": [ 331 331 + "FakeSchemaPackable(property_b='', property_a=5)" 332 332 + ] 333 333 + }, 334 334 + "execution_count": 15, 335 335 + "metadata": {}, 336 336 + "output_type": "execute_result" 337 337 + } 338 338 + ], 339 339 + "source": [ 340 340 + "FakeSchemaPackable.from_bytes( x.as_wds['msgpack'] )" 341 341 + ] 342 342 + }, 343 343 + { 344 344 + "cell_type": "code", 345 345 + "execution_count": 18, 346 346 + "id": "0e97b846", 347 347 + "metadata": {}, 348 348 + "outputs": [], 349 349 + "source": [ 350 350 + "xw = x.as_wds" 351 351 + ] 352 352 + }, 353 353 + { 354 354 + "cell_type": "code", 355 355 + "execution_count": 19, 356 356 + "id": "ff625d70", 357 357 + "metadata": {}, 358 358 + "outputs": [ 359 359 + { 360 360 + "data": { 361 361 + "text/plain": [ 362 362 + "{'__key__': '36e2f3c0-cb2e-11f0-8000-000000000000', 'msgpack': b'\\x80'}" 363 363 + ] 364 364 + }, 365 365 + "execution_count": 19, 366 366 + "metadata": {}, 367 367 + "output_type": "execute_result" 368 368 + } 369 369 + ], 370 370 + "source": [ 371 371 + "xw" 372 372 + ] 373 373 + } 374 374 + ], 375 375 + "metadata": { 376 376 + "kernelspec": { 377 377 + "display_name": "atdata", 378 378 + "language": "python", 379 379 + "name": "python3" 380 380 + }, 381 381 + "language_info": { 382 382 + "codemirror_mode": { 383 383 + "name": "ipython", 384 384 + "version": 3 385 385 + }, 386 386 + "file_extension": ".py", 387 387 + "mimetype": "text/x-python", 388 388 + "name": "python", 389 389 + "nbconvert_exporter": "python", 390 390 + "pygments_lexer": "ipython3", 391 391 + "version": "3.12.11" 392 392 + } 393 393 + }, 394 394 + "nbformat": 4, 395 395 + "nbformat_minor": 5 396 396 + }