bpev.me / civility
A system for building static webapps
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat(ai): add ai package

Ben Pevsner f822b243 2f6ac53f

+313 -45
+3 -1
deno.lock
··· 23 23 "jsr:@rodney/parsedown@^1.4.3": "1.4.3", 24 24 "jsr:@std/assert@^1.0.19": "1.0.19", 25 25 "jsr:@std/async@1": "1.3.0", 26 + "jsr:@std/async@^1.3.0": "1.3.0", 26 27 "jsr:@std/bytes@^1.0.2": "1.0.6", 27 28 "jsr:@std/cli@^1.0.29": "1.0.29", 28 29 "jsr:@std/collections@^1.1.3": "1.1.7", ··· 89 90 "dependencies": [ 90 91 "jsr:@deno-library/progress", 91 92 "jsr:@deno/cache-dir", 92 - "jsr:@std/async", 93 + "jsr:@std/async@1", 93 94 "jsr:@std/encoding@1", 94 95 "jsr:@std/fs@1", 95 96 "jsr:@std/path@1", ··· 314 315 "integrity": "d3152f57b11666bf6358d0e127c7e3488e91178b0c2d8fbf0793e1c53cd13cb1", 315 316 "dependencies": [ 316 317 "jsr:@std/assert", 318 + "jsr:@std/async@^1.3.0", 317 319 "jsr:@std/data-structures", 318 320 "jsr:@std/fs@^1.0.23", 319 321 "jsr:@std/internal@^1.0.13",
+4 -4
docs/spec/storage.md
··· 713 713 714 714 **Resolved:** 715 715 716 - | Feature | Notes | 717 - | ---------------------------------- | ------------------------------------------------------------------------- | 718 - | False-positive conflict detection | `pathsConflict()` replaces `topLevelPath()`; full-path prefix comparison | 719 - | S3-compatible `ObjectStorage` | `S3Storage` in `packages/hono/storage/s3.ts`; `list()` included | 716 + | Feature | Notes | 717 + | --------------------------------- | ------------------------------------------------------------------------ | 718 + | False-positive conflict detection | `pathsConflict()` replaces `topLevelPath()`; full-path prefix comparison | 719 + | S3-compatible `ObjectStorage` | `S3Storage` in `packages/hono/storage/s3.ts`; `list()` included |
+39 -39
docs/spec/sync.md
··· 540 540 541 541 ### 9.3 Server Store Backends (`SyncStorage`) 542 542 543 - | Backend | Module | Status | 544 - | ---------- | ----------------------------- | ---------------------------------------- | 545 - | Deno KV | `@civility/hono/sync/deno-kv` | ✓ | 543 + | Backend | Module | Status | 544 + | ---------- | ----------------------------- | ----------------------------------------- | 545 + | Deno KV | `@civility/hono/sync/deno-kv` | ✓ | 546 546 | PostgreSQL | — | ⚠️ planned (Phase 5; may pull to Phase 4) | 547 - | SQLite | — | ⚠️ planned (Phase 5) | 547 + | SQLite | — | ⚠️ planned (Phase 5) | 548 548 549 549 ### 9.4 Server Object Storage (`ObjectStorage`) 550 550 ··· 558 558 getUrl(key: string): Promise<string | null> // signed/CDN URL for direct client download 559 559 has(key: string): Promise<boolean> 560 560 delete(key: string): Promise<void> 561 - list(): AsyncIterableIterator<BlobMeta> // used for blob GC (§8.6) 561 + list(): AsyncIterableIterator<BlobMeta> // used for blob GC (§8.6) 562 562 } 563 563 ``` 564 564 ··· 566 566 567 567 **Known coupling ⚠️:** Both `BunnyStorage` and `S3Storage` currently require `kv: Deno.Kv` in their config and write blob metadata (`BlobMeta`) directly to KV under `['blob-meta', key]`. This is an internal implementation concern — the `ObjectStorage` interface does not expose KV — but it means object storage adapters cannot function without Deno KV. Metadata ownership should move to `SyncStorage` so adapters are self-contained. See §13 open gaps. 568 568 569 - | Backend | Module | Status | 570 - | ------- | ------ | ------ | 571 - | Bunny Storage | `@civility/hono/storage/bunny` | ✓ (deprecation planned — see below) | 572 - | S3-compatible (AWS S3, Cloudflare R2, MinIO, Bunny) | `@civility/hono/storage/s3` via `@bradenmacdonald/s3-lite-client` | ✓ | 573 - | Local filesystem | — | ⚠️ planned (Phase 5) | 569 + | Backend | Module | Status | 570 + | --------------------------------------------------- | ----------------------------------------------------------------- | ----------------------------------- | 571 + | Bunny Storage | `@civility/hono/storage/bunny` | ✓ (deprecation planned — see below) | 572 + | S3-compatible (AWS S3, Cloudflare R2, MinIO, Bunny) | `@civility/hono/storage/s3` via `@bradenmacdonald/s3-lite-client` | ✓ | 573 + | Local filesystem | — | ⚠️ planned (Phase 5) | 574 574 575 575 **Bunny deprecation:** Bunny now supports the S3-compatible API. `S3Storage` configured against Bunny's S3 endpoint is the intended long-term default. `BunnyStorage` (which uses Bunny's proprietary HTTP API) will be deprecated once `S3Storage` is validated as the replacement. No migration is needed — both backends hash keys and use KV for metadata identically. 576 576 ··· 664 664 665 665 **Open gaps (ordered by priority):** 666 666 667 - | # | Gap | Severity | Notes | 668 - | -- | ------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------ | 669 - | 1 | Server-side blob GC | High | `ObjectStorage.list()` now exists; sweep endpoint still needed | 670 - | 2 | Client-side blob GC | High | `@civility/blobs` compaction flow; Phase 4 | 671 - | 3 | Blob metadata KV coupling | High | `S3Storage`/`BunnyStorage` write metadata to Deno KV directly; should move to `SyncStorage` so adapters are KV-agnostic | 672 - | 4 | BunnyStorage deprecation | Medium | Replace with `S3Storage` against Bunny S3 endpoint; remove proprietary Bunny HTTP adapter | 673 - | 5 | E2E encryption implementation | Medium | Architecture complete in `encryption.md`; Phase 4 | 674 - | 6 | Scheduled change log pruning | Medium | Currently admin-triggered only; needs cron facility | 675 - | 7 | PostgreSQL `SyncStorage` adapter | Medium | Phase 5; may pull forward; key for self-hosted SQL deployments | 676 - | 8 | Local filesystem `ObjectStorage` | Medium | Important for self-hosted dev workflows | 677 - | 9 | SQLite `SyncStorage` adapter | Low | Phase 5; lightweight self-hosted option | 678 - | 10 | Pluggable per-field merge strategies | Low | `array-union`, `map-merge`; Phase 5 | 679 - | 11 | Real-time WebSocket sync | Low | Phase 5; polling sufficient for single-user | 680 - | 12 | Undo/redo stack | Low | Inverse patches exist in store; Phase 5 | 667 + | # | Gap | Severity | Notes | 668 + | -- | ------------------------------------ | -------- | ----------------------------------------------------------------------------------------------------------------------- | 669 + | 1 | Server-side blob GC | High | `ObjectStorage.list()` now exists; sweep endpoint still needed | 670 + | 2 | Client-side blob GC | High | `@civility/blobs` compaction flow; Phase 4 | 671 + | 3 | Blob metadata KV coupling | High | `S3Storage`/`BunnyStorage` write metadata to Deno KV directly; should move to `SyncStorage` so adapters are KV-agnostic | 672 + | 4 | BunnyStorage deprecation | Medium | Replace with `S3Storage` against Bunny S3 endpoint; remove proprietary Bunny HTTP adapter | 673 + | 5 | E2E encryption implementation | Medium | Architecture complete in `encryption.md`; Phase 4 | 674 + | 6 | Scheduled change log pruning | Medium | Currently admin-triggered only; needs cron facility | 675 + | 7 | PostgreSQL `SyncStorage` adapter | Medium | Phase 5; may pull forward; key for self-hosted SQL deployments | 676 + | 8 | Local filesystem `ObjectStorage` | Medium | Important for self-hosted dev workflows | 677 + | 9 | SQLite `SyncStorage` adapter | Low | Phase 5; lightweight self-hosted option | 678 + | 10 | Pluggable per-field merge strategies | Low | `array-union`, `map-merge`; Phase 5 | 679 + | 11 | Real-time WebSocket sync | Low | Phase 5; polling sufficient for single-user | 680 + | 12 | Undo/redo stack | Low | Inverse patches exist in store; Phase 5 | 681 681 682 682 **Resolved:** 683 683 684 - | Feature | Phase | Notes | 685 - | --------------------------------------------------------------- | ----- | ----------------------------------------------------------------------- | 686 - | `@civility/store` (HLC, JSON Patch, merge, schema, history) | 1 | Core store foundation, 290+ tests | 687 - | MVP: `@civility/blobs`, `@civility/sync`, `@civility/hono`, CLI | 2 | End-to-end sync working across apps | 688 - | Structured error system (`@civility/errors`) | 3.1 | `CivilityApiError`, `SyncErrorEvent`, envelope helpers | 689 - | Store properties, entity detection from schema | 3.2 | `Property<T>`, `_civ:` namespace, combined push requests | 690 - | Server-side schema validation | 3.3 | Inline validator; compatibility checks on registration | 691 - | App permission model | 3.4 | Manifest, grants, popup auth flow, audit log | 692 - | User limits and resource management | 3.5 | Quotas, rate limits, change log pruning, selective sync | 693 - | Import/export semantics | 3.6 | `replace`/`extend`, force push/pull, server backup/restore | 694 - | UI components | 3.7 | `ui-sync-input`, `ui-sync-state`, `ui-store-import`, `ui-error-display` | 695 - | Merge strategy research | 3.8 | LWW + three-way merge is sufficient; see `merge_strategy.md` | 696 - | E2E encryption design | 3.9 | Wrapped-key architecture; see `encryption.md` | 697 - | Auth hardening (oslo) | 3.10 | TOTP MFA, sessions, tokens, rate limiting; see `auth.md` | 684 + | Feature | Phase | Notes | 685 + | --------------------------------------------------------------- | ----- | ------------------------------------------------------------------------ | 686 + | `@civility/store` (HLC, JSON Patch, merge, schema, history) | 1 | Core store foundation, 290+ tests | 687 + | MVP: `@civility/blobs`, `@civility/sync`, `@civility/hono`, CLI | 2 | End-to-end sync working across apps | 688 + | Structured error system (`@civility/errors`) | 3.1 | `CivilityApiError`, `SyncErrorEvent`, envelope helpers | 689 + | Store properties, entity detection from schema | 3.2 | `Property<T>`, `_civ:` namespace, combined push requests | 690 + | Server-side schema validation | 3.3 | Inline validator; compatibility checks on registration | 691 + | App permission model | 3.4 | Manifest, grants, popup auth flow, audit log | 692 + | User limits and resource management | 3.5 | Quotas, rate limits, change log pruning, selective sync | 693 + | Import/export semantics | 3.6 | `replace`/`extend`, force push/pull, server backup/restore | 694 + | UI components | 3.7 | `ui-sync-input`, `ui-sync-state`, `ui-store-import`, `ui-error-display` | 695 + | Merge strategy research | 3.8 | LWW + three-way merge is sufficient; see `merge_strategy.md` | 696 + | E2E encryption design | 3.9 | Wrapped-key architecture; see `encryption.md` | 697 + | Auth hardening (oslo) | 3.10 | TOTP MFA, sessions, tokens, rate limiting; see `auth.md` | 698 698 | False-positive conflict detection fix | 3 | `pathsConflict()` replaces `topLevelPath()`; full-path prefix comparison | 699 - | S3-compatible `ObjectStorage` adapter | 3 | `S3Storage` in `packages/hono/storage/s3.ts`; `list()` included | 699 + | S3-compatible `ObjectStorage` adapter | 3 | `S3Storage` in `packages/hono/storage/s3.ts`; `list()` included |
+34
packages/ai/README.md
··· 1 + # bpev AI 2 + 3 + For an Image 4 + 5 + ``` 6 + curl -X POST "https://your-deno-deploy-project.deno.dev/process-input" \ 7 + -H "Content-Type: application/json" \ 8 + -d '{ 9 + "type": "image", 10 + "content": "your_base64_encoded_image_data" 11 + }' 12 + ``` 13 + 14 + For a Text File 15 + 16 + ``` 17 + curl -X POST "https://your-deno-deploy-project.deno.dev/process-input" \ 18 + -H "Content-Type: application/json" \ 19 + -d '{ 20 + "type": "text", 21 + "content": "/path/to/your/textfile.txt" 22 + }' 23 + ``` 24 + 25 + For a URL 26 + 27 + ``` 28 + curl -X POST "https://your-deno-deploy-project.deno.dev/process-input" \ 29 + -H "Content-Type: application/json" \ 30 + -d '{ 31 + "type": "url", 32 + "content": "https://example.com/your-recipe-page" 33 + }' 34 + ```
+7
packages/ai/__tests__/mod.test.ts
··· 1 + import { createClient } from '../mod.ts' 2 + import { assertRejects } from '@std/assert' 3 + 4 + Deno.test('initialize client', async () => { 5 + const client = createClient({ apiKey: 'blahblah-api-key' }) 6 + await assertRejects(() => client.generateText({ prompt: '...' })) 7 + })
+53
packages/ai/apis/claude.ts
··· 1 + /** Configuration for the Claude API client */ 2 + export interface Config { 3 + apiKey: string 4 + proxyUrl?: string 5 + model?: string 6 + maxTokens?: number 7 + } 8 + 9 + /** Response from Claude API */ 10 + interface Response { 11 + content: Array<{ type: 'text'; text: string }> 12 + } 13 + 14 + /** Default configuration values */ 15 + export const DEFAULT_CONFIG = { 16 + proxyUrl: 'https://ai.bpev.me', 17 + model: 'claude-sonnet-4-5', 18 + maxTokens: 4096, 19 + } 20 + 21 + /** Call the Claude API through the proxy */ 22 + export async function invoke( 23 + prompt: string, 24 + { model, maxTokens, apiKey, proxyUrl }: Required<Config>, 25 + systemPrompt?: string, 26 + ): Promise<string> { 27 + const body: Record<string, unknown> = { 28 + model, 29 + max_tokens: maxTokens, 30 + messages: [{ role: 'user' as const, content: prompt }], 31 + } 32 + 33 + if (systemPrompt) body.system = systemPrompt 34 + 35 + const response = await fetch(`${proxyUrl}/proxy/claude/v1/messages`, { 36 + method: 'POST', 37 + headers: { 38 + 'Content-Type': 'application/json', 39 + 'x-claude-key': apiKey, 40 + }, 41 + body: JSON.stringify(body), 42 + }) 43 + 44 + if (!response.ok) { 45 + const error = await response.json().catch(() => ({})) 46 + throw new Error( 47 + error.error || `HTTP ${response.status}: ${response.statusText}`, 48 + ) 49 + } 50 + 51 + const data = await response.json() as Response 52 + return data.content[0]?.text || '' 53 + }
+8
packages/ai/deno.json
··· 1 + { 2 + "exports": { 3 + ".": "./mod.ts" 4 + }, 5 + "imports": { 6 + "@zod/zod": "jsr:@zod/zod@^4.3.6" 7 + } 8 + }
+115
packages/ai/mod.ts
··· 1 + import { z } from '@zod/zod' 2 + import { 3 + type Config as ClaudeConfig, 4 + DEFAULT_CONFIG as CLAUDE_DEFAULT_CONFIG, 5 + invoke, 6 + } from './apis/claude.ts' 7 + import { buildPrompt } from './utils/build_prompt.ts' 8 + import { parseJSON } from './utils/parse_json.ts' 9 + 10 + /** 11 + * Create a configured client for easier repeated calls 12 + * 13 + * @example 14 + * ```ts 15 + * const client = createClient({ apiKey: "sk-..." }) 16 + * const result1 = await client.generateStructured({ prompt: "...", schema: Schema1 }) 17 + * const result2 = await client.generateStructured({ prompt: "...", schema: Schema2 }) 18 + * ``` 19 + */ 20 + export function createClient<T extends z.ZodType>(config: ClaudeConfig): { 21 + generateStructured: ( 22 + options: Omit<StructureOptions<T>, 'config'>, 23 + ) => Promise<z.infer<T>> 24 + 25 + generateText: ( 26 + options: Omit<Parameters<typeof generateText>[0], 'config'>, 27 + ) => Promise<string> 28 + } { 29 + return { 30 + generateStructured: ( 31 + options: Omit<StructureOptions<T>, 'config'>, 32 + ) => generateStructured({ ...options, config }), 33 + 34 + generateText: ( 35 + options: Omit<Parameters<typeof generateText>[0], 'config'>, 36 + ) => generateText({ ...options, config }), 37 + } 38 + } 39 + 40 + /** 41 + * Simple text generation without schema validation 42 + * 43 + * @example 44 + * ```ts 45 + * const response = await generateText({ 46 + * prompt: "Explain quantum computing", 47 + * config: { apiKey: "sk-..." } 48 + * }) 49 + * ``` 50 + */ 51 + export function generateText(options: { 52 + prompt: string 53 + config: ClaudeConfig 54 + systemPrompt?: string 55 + }): Promise<string> { 56 + const config: Required<ClaudeConfig> = { 57 + apiKey: options.config.apiKey, 58 + proxyUrl: options.config.proxyUrl ?? CLAUDE_DEFAULT_CONFIG.proxyUrl, 59 + model: options.config.model ?? CLAUDE_DEFAULT_CONFIG.model, 60 + maxTokens: options.config.maxTokens ?? CLAUDE_DEFAULT_CONFIG.maxTokens, 61 + } 62 + 63 + if (!config.apiKey) throw new Error('API key is required') 64 + 65 + return invoke(options.prompt, config, options.systemPrompt) 66 + } 67 + 68 + /** Options for generating structured output */ 69 + export interface StructureOptions<T extends z.ZodType> { 70 + prompt: string 71 + schema: T 72 + userDescription?: string 73 + config: ClaudeConfig 74 + systemPrompt?: string 75 + } 76 + 77 + /** 78 + * Generate structured output from Claude using a Zod schema 79 + * 80 + * @example 81 + * ```ts 82 + * const result = await generateStructured({ 83 + * prompt: "Create a workout template", 84 + * schema: SessionTemplate, 85 + * userDescription: "30 minute full body workout", 86 + * config: { apiKey: "sk-..." } 87 + * }) 88 + * ``` 89 + */ 90 + export async function generateStructured<T extends z.ZodType>( 91 + options: StructureOptions<T>, 92 + ): Promise<z.infer<T>> { 93 + const { prompt, schema, userDescription, systemPrompt } = options 94 + const config: Required<ClaudeConfig> = { 95 + apiKey: options.config.apiKey, 96 + proxyUrl: options.config.proxyUrl ?? CLAUDE_DEFAULT_CONFIG.proxyUrl, 97 + model: options.config.model ?? CLAUDE_DEFAULT_CONFIG.model, 98 + maxTokens: options.config.maxTokens ?? CLAUDE_DEFAULT_CONFIG.maxTokens, 99 + } 100 + 101 + if (!config.apiKey) throw new Error('API key is required') 102 + 103 + try { 104 + const fullPrompt = buildPrompt(prompt, schema, userDescription) 105 + const responseText = await invoke(fullPrompt, config, systemPrompt) 106 + return schema.parse(parseJSON(responseText)) 107 + } catch (error) { 108 + console.error('Error generating structured output:', error) 109 + throw new Error( 110 + `Failed to generate structured output: ${ 111 + error instanceof Error ? error.message : 'Unknown error' 112 + }`, 113 + ) 114 + } 115 + }
+29
packages/ai/utils/build_prompt.ts
··· 1 + import z from '@zod/zod' 2 + 3 + /** 4 + * Generate format instructions from a Zod schema 5 + */ 6 + function generateFormatInstructions<T extends z.ZodType>(schema: T): string { 7 + return `You must respond with valid JSON that matches this schema:\n${ 8 + JSON.stringify(z.toJSONSchema(schema), null, 2) 9 + }` 10 + } 11 + 12 + /** 13 + * Build the complete prompt with schema instructions 14 + */ 15 + export function buildPrompt<T extends z.ZodType>( 16 + prompt: string, 17 + schema: T, 18 + userDescription?: string, 19 + ): string { 20 + let fullPrompt = prompt 21 + 22 + if (userDescription) { 23 + fullPrompt += `\n\nUser request: ${userDescription}` 24 + } 25 + 26 + fullPrompt += `\n\n${generateFormatInstructions(schema)}` 27 + 28 + return fullPrompt 29 + }
+18
packages/ai/utils/parse_json.ts
··· 1 + /** 2 + * Parse JSON from Claude's response, handling various formats 3 + */ 4 + export function parseJSON(text: string): unknown { 5 + try { 6 + return JSON.parse(text) 7 + } catch { 8 + // Try to extract JSON from markdown code blocks 9 + const jsonMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/) 10 + if (jsonMatch) return JSON.parse(jsonMatch[1]) 11 + 12 + // Try to find any JSON object in the response 13 + const objectMatch = text.match(/\{[\s\S]*\}/) 14 + if (objectMatch) return JSON.parse(objectMatch[0]) 15 + 16 + throw new Error('Could not parse JSON from response') 17 + } 18 + }
+3 -1
packages/hono/main.ts
··· 15 15 const email = smtpHost 16 16 ? createSmtpAdapter({ 17 17 host: smtpHost, 18 - port: Deno.env.get('SMTP_PORT') ? Number(Deno.env.get('SMTP_PORT')) : undefined, 18 + port: Deno.env.get('SMTP_PORT') 19 + ? Number(Deno.env.get('SMTP_PORT')) 20 + : undefined, 19 21 secure: Deno.env.get('SMTP_SECURE') === 'true', 20 22 username: Deno.env.get('SMTP_USER'), 21 23 password: Deno.env.get('SMTP_PASS'),