Replace SSIM-based frame cropping with full-frame RMS comparison

+1 -3

JOURNAL.md

··· 568 568 569 569 #### Screen frame extracts 570 570 571 - Screen analysis files use per-monitor naming: `<position>_<connector>_screen.jsonl` (e.g., `center_DP-3_screen.jsonl`, `left_HDMI-1_screen.jsonl`). For single-monitor setups, the file is simply `screen.jsonl`. Each file contains one JSON object per qualified frame. Frames qualify when they contain a changed region of at least 400×400 pixels, detected using block-based SSIM comparison. 571 + Screen analysis files use per-monitor naming: `<position>_<connector>_screen.jsonl` (e.g., `center_DP-3_screen.jsonl`, `left_HDMI-1_screen.jsonl`). For single-monitor setups, the file is simply `screen.jsonl`. Each file contains one JSON object per qualified frame. Frames qualify when they show significant visual change (≥5% RMS difference) compared to the previous qualified frame. 572 572 573 573 Example frame record: 574 574 ··· 576 576 { 577 577 "frame_id": 123, 578 578 "timestamp": 45.67, 579 - "box_2d": [100, 200, 500, 600], 580 579 "requests": [ 581 580 {"type": "describe_json", "model": "gemini-2.0-flash-lite", "duration": 0.5} 582 581 ], ··· 590 589 **Common fields:** 591 590 - `frame_id` – sequential frame number in the video 592 591 - `timestamp` – time in seconds from video start 593 - - `box_2d` – bounding box of changed region `[x_min, y_min, x_max, y_max]` 594 592 - `requests` – list of vision API requests made for this frame 595 593 - `analysis` – categorization and visual description from initial analysis 596 594

+3 -3

fixtures/journal/20240102/234567_300/screen.jsonl

··· 1 1 {"raw": "screen.webm"} 2 - {"frame_id": 1, "timestamp": 5.2, "monitor": "DP-1", "monitor_position": "center", "box_2d": [100, 200, 900, 1200], "requests": [{"type": "describe_json", "model": "gemini-2.0-flash-lite", "duration": 0.4}], "analysis": {"visual_description": "VSCode IDE window with Python code open, showing the FastAPI application auth module with breakpoints set.", "visible": "code"}, "extracted_text": "class TokenValidator:\n def validate_token(self, token: str) -> bool:\n # Breakpoint here\n decoded = jwt.decode(token)"} 3 - {"frame_id": 2, "timestamp": 15.8, "monitor": "DP-1", "monitor_position": "center", "box_2d": [150, 250, 850, 1100], "requests": [{"type": "describe_json", "model": "gemini-2.0-flash-lite", "duration": 0.5}], "analysis": {"visual_description": "Terminal window showing Docker logs from the FastAPI container, displaying request logs and database queries.", "visible": "terminal"}, "extracted_text": "fastapi_1 | INFO: 192.168.1.100:54321 - 'GET /auth/validate HTTP/1.1' 200 OK\npostgres_1 | LOG: duration: 2.451 ms statement: SELECT * FROM users WHERE id = $1"} 4 - {"frame_id": 3, "timestamp": 28.3, "monitor": "DP-1", "monitor_position": "center", "box_2d": [200, 300, 950, 1250], "requests": [{"type": "describe_json", "model": "gemini-2.0-flash-lite", "duration": 0.3}], "analysis": {"visual_description": "Split terminal view with pytest output on the left and PostgreSQL query logs on the right.", "visible": "terminal"}, "extracted_text": "============================= test session starts ==============================\ntests/test_auth.py::test_token_validation PASSED\ntests/test_auth.py::test_timezone_handling FAILED"} 2 + {"frame_id": 1, "timestamp": 5.2, "monitor": "DP-1", "monitor_position": "center", "requests": [{"type": "describe_json", "model": "gemini-2.0-flash-lite", "duration": 0.4}], "analysis": {"visual_description": "VSCode IDE window with Python code open, showing the FastAPI application auth module with breakpoints set.", "visible": "code"}, "extracted_text": "class TokenValidator:\n def validate_token(self, token: str) -> bool:\n # Breakpoint here\n decoded = jwt.decode(token)"} 3 + {"frame_id": 2, "timestamp": 15.8, "monitor": "DP-1", "monitor_position": "center", "requests": [{"type": "describe_json", "model": "gemini-2.0-flash-lite", "duration": 0.5}], "analysis": {"visual_description": "Terminal window showing Docker logs from the FastAPI container, displaying request logs and database queries.", "visible": "terminal"}, "extracted_text": "fastapi_1 | INFO: 192.168.1.100:54321 - 'GET /auth/validate HTTP/1.1' 200 OK\npostgres_1 | LOG: duration: 2.451 ms statement: SELECT * FROM users WHERE id = $1"} 4 + {"frame_id": 3, "timestamp": 28.3, "monitor": "DP-1", "monitor_position": "center", "requests": [{"type": "describe_json", "model": "gemini-2.0-flash-lite", "duration": 0.3}], "analysis": {"visual_description": "Split terminal view with pytest output on the left and PostgreSQL query logs on the right.", "visible": "terminal"}, "extracted_text": "============================= test session starts ==============================\ntests/test_auth.py::test_token_validation PASSED\ntests/test_auth.py::test_timezone_handling FAILED"}

+88 -235

observe/describe.py

··· 3 3 Describe screencast videos by detecting significant frame changes. 4 4 5 5 Processes per-monitor screencast files (.webm/.mp4/.mov), detects changes using 6 - block-based SSIM, and qualifies frames that meet the 400x400 threshold for 7 - Gemini processing. 6 + RMS-based comparison, and sends full frames to Gemini for analysis. 8 7 """ 9 8 10 9 from __future__ import annotations ··· 21 20 from typing import List, Optional 22 21 23 22 import av 24 - import numpy as np 25 - from PIL import Image 23 + from PIL import Image, ImageChops, ImageStat 26 24 27 25 from think.callosum import callosum_send 28 26 from think.utils import setup_cli ··· 77 75 class VideoProcessor: 78 76 """Process per-monitor screencast videos and detect significant frame changes.""" 79 77 78 + # RMS threshold for frame qualification (5% difference) 79 + RMS_THRESHOLD = 0.05 80 + # Downsample size for RMS comparison 81 + COMPARE_SIZE = (160, 90) 82 + 80 83 def __init__(self, video_path: Path): 81 84 self.video_path = video_path 82 85 self.width: Optional[int] = None ··· 92 95 """ 93 96 Process video and return qualified frames. 94 97 98 + Uses RMS-based comparison on downsampled frames to detect significant 99 + changes. Caches the downsampled version of the last qualified frame 100 + to avoid repeated resizing. 101 + 95 102 Returns: 96 - List of qualified frames with timestamp, frame data (as bytes), 97 - and change boxes. 103 + List of qualified frames with timestamp and frame_bytes. 98 104 """ 99 - last_qualified: Optional[np.ndarray] = None 105 + # Cache for downsampled last qualified frame 106 + last_qualified_small: Optional[Image.Image] = None 100 107 101 108 try: 102 109 with av.open(str(self.video_path)) as container: ··· 114 121 timestamp = frame.time if frame.time is not None else 0.0 115 122 frame_count += 1 116 123 117 - # First frame: always qualify with full frame bounds 118 - if last_qualified is None: 119 - box_2d = [0, 0, self.height, self.width] 120 - 121 - # Convert to PIL for bytes conversion 122 - arr_rgb = frame.to_ndarray(format="rgb24") 123 - pil_img = Image.fromarray(arr_rgb) 124 + # Convert to PIL for comparison and bytes conversion 125 + arr_rgb = frame.to_ndarray(format="rgb24") 126 + pil_img = Image.fromarray(arr_rgb) 127 + del arr_rgb 124 128 125 - # Convert frame to bytes immediately 126 - crop_bytes = self._frame_to_crop_bytes(pil_img, box_2d) 129 + # Downsample for comparison 130 + current_small = self._downsample(pil_img) 127 131 128 - # Clean up PIL image and RGB array 132 + # First frame: always qualify (RMS vs nothing = 100% different) 133 + if last_qualified_small is None: 134 + frame_bytes = self._frame_to_bytes(pil_img) 129 135 pil_img.close() 130 - del arr_rgb 131 136 132 137 self.qualified_frames.append( 133 138 { 134 139 "frame_id": frame_count, 135 140 "timestamp": timestamp, 136 - "box_2d": box_2d, 137 - "crop_bytes": crop_bytes, 141 + "frame_bytes": frame_bytes, 138 142 } 139 143 ) 140 144 141 - # Store grayscale numpy array for comparison 142 - last_qualified = frame.to_ndarray(format="gray") 143 - 145 + last_qualified_small = current_small 144 146 logger.debug(f"First frame at {timestamp:.2f}s") 145 147 continue 146 148 147 - # Compare current frame with last qualified 148 - frame_gray = frame.to_ndarray(format="gray") 149 - boxes = self._compare_frames(last_qualified, frame_gray) 149 + # Compare current frame with last qualified using RMS 150 + rms = self._rms_diff(last_qualified_small, current_small) 150 151 151 - if not boxes: 152 + if rms < self.RMS_THRESHOLD: 153 + # Not enough change - skip this frame 154 + current_small.close() 155 + pil_img.close() 152 156 continue 153 157 154 - # Find largest box by area 155 - largest_box = max( 156 - boxes, 157 - key=lambda b: (b["box_2d"][2] - b["box_2d"][0]) 158 - * (b["box_2d"][3] - b["box_2d"][1]), 159 - ) 160 - 161 - y_min, x_min, y_max, x_max = largest_box["box_2d"] 162 - box_width = x_max - x_min 163 - box_height = y_max - y_min 164 - 165 - # Qualify if largest box meets threshold 166 - if box_width >= 400 and box_height >= 400: 167 - arr = frame.to_ndarray(format="rgb24") 168 - frame_pil = Image.fromarray(arr) 169 - del arr 170 - 171 - # Convert frame to bytes immediately 172 - crop_bytes = self._frame_to_crop_bytes( 173 - frame_pil, largest_box["box_2d"] 174 - ) 158 + # Qualified - convert full frame to bytes 159 + frame_bytes = self._frame_to_bytes(pil_img) 160 + pil_img.close() 175 161 176 - frame_pil.close() 162 + self.qualified_frames.append( 163 + { 164 + "frame_id": frame_count, 165 + "timestamp": timestamp, 166 + "frame_bytes": frame_bytes, 167 + } 168 + ) 177 169 178 - self.qualified_frames.append( 179 - { 180 - "frame_id": frame_count, 181 - "timestamp": timestamp, 182 - "box_2d": largest_box["box_2d"], 183 - "crop_bytes": crop_bytes, 184 - } 185 - ) 170 + # Update cached downsampled frame 171 + last_qualified_small.close() 172 + last_qualified_small = current_small 186 173 187 - # Store grayscale numpy array for comparison 188 - last_qualified = frame_gray 174 + logger.debug( 175 + f"Qualified frame at {timestamp:.2f}s (RMS: {rms:.3f})" 176 + ) 189 177 190 - logger.debug( 191 - f"Qualified frame at {timestamp:.2f}s " 192 - f"(box: {box_width}x{box_height})" 193 - ) 178 + # Clean up last cached frame 179 + if last_qualified_small is not None: 180 + last_qualified_small.close() 194 181 195 182 logger.info( 196 183 f"Processed {frame_count} frames from {self.video_path.name}, " ··· 205 192 206 193 return self.qualified_frames 207 194 208 - def _frame_to_crop_bytes( 209 - self, 210 - img: Image.Image, 211 - box_2d: list, 212 - ) -> bytes: 195 + def _downsample(self, img: Image.Image) -> Image.Image: 196 + """Downsample image to comparison size.""" 197 + if img.mode != "RGB": 198 + img = img.convert("RGB") 199 + return img.resize(self.COMPARE_SIZE, Image.BILINEAR) 200 + 201 + def _rms_diff(self, img1: Image.Image, img2: Image.Image) -> float: 213 202 """ 214 - Convert cropped change region to PNG bytes. 215 - 216 - Parameters 217 - ---------- 218 - img : Image.Image 219 - PIL Image to convert (full frame) 220 - box_2d : list 221 - Change box [y_min, x_min, y_max, x_max] 203 + Compute RMS difference between two images, normalized to [0, 1]. 222 204 223 - Returns 224 - ------- 225 - bytes 226 - Crop image as PNG bytes 205 + Both images should already be downsampled to COMPARE_SIZE. 227 206 """ 228 - y_min, x_min, y_max, x_max = box_2d 229 - 230 - # Expand bounds by 50px in all directions where possible 231 - img_width, img_height = img.size 232 - expanded_x_min = max(0, x_min - 50) 233 - expanded_y_min = max(0, y_min - 50) 234 - expanded_x_max = min(img_width, x_max + 50) 235 - expanded_y_max = min(img_height, y_max + 50) 236 - 237 - # Crop to expanded region 238 - cropped = img.crop( 239 - (expanded_x_min, expanded_y_min, expanded_x_max, expanded_y_max) 240 - ) 207 + diff = ImageChops.difference(img1, img2) 208 + stat = ImageStat.Stat(diff) 209 + # Normalize RMS to [0, 1] by dividing by 255 per channel 210 + rms = sum(stat.rms) / (len(stat.rms) * 255.0) 211 + return float(rms) 241 212 242 - # Convert to PNG bytes (compress_level=1 for speed) 243 - crop_io = io.BytesIO() 244 - cropped.save(crop_io, format="PNG", compress_level=1) 245 - crop_bytes = crop_io.getvalue() 246 - crop_io.close() 247 - cropped.close() 248 - 249 - # img is closed by caller 250 - 251 - return crop_bytes 252 - 253 - def _load_full_frame(self, frame_id: int) -> Image.Image: 254 - """Load a full frame by 1-based frame_id for meeting analysis.""" 255 - with av.open(str(self.video_path)) as container: 256 - stream = container.streams.video[0] 257 - stream.thread_type = "AUTO" 258 - stream.codec_context.thread_count = 0 259 - 260 - frame_count = 0 261 - for frame in container.decode(video=0): 262 - if frame.pts is None: 263 - continue 264 - 265 - frame_count += 1 266 - if frame_count == frame_id: 267 - arr = frame.to_ndarray(format="rgb24") 268 - return Image.fromarray(arr) 269 - 270 - raise ValueError( 271 - f"Frame {frame_id} not found in {self.video_path}" 272 - ) 273 - 274 - def _compare_frames( 275 - self, 276 - frame1: np.ndarray, 277 - frame2: np.ndarray, 278 - mad_threshold: float = 1.5, 279 - ) -> List[dict]: 213 + def _frame_to_bytes(self, img: Image.Image) -> bytes: 280 214 """ 281 - Compare two grayscale frames and return changed region boxes. 215 + Convert full frame to PNG bytes. 282 216 283 217 Parameters 284 218 ---------- 285 - frame1 : np.ndarray 286 - First grayscale frame 287 - frame2 : np.ndarray 288 - Second grayscale frame 289 - mad_threshold : float 290 - Mean Absolute Difference threshold for early bailout (default: 1.5) 291 - If downsampled MAD is below this, skip expensive SSIM computation 219 + img : Image.Image 220 + PIL Image to convert 292 221 293 222 Returns 294 223 ------- 295 - List[dict] 296 - Boxes with 'box_2d' key containing [y_min, x_min, y_max, x_max] 297 - """ 298 - # Fast pre-filter: compute MAD on 1/4-scale downsampled images 299 - # This is extremely fast (pure NumPy) and eliminates unchanged frames 300 - small1 = frame1[::4, ::4].astype(np.int16) 301 - small2 = frame2[::4, ::4].astype(np.int16) 302 - mad = np.abs(small1 - small2).mean() 303 - 304 - if mad < mad_threshold: 305 - # No significant change detected - skip expensive SSIM 306 - return [] 307 - 308 - return self._compute_ssim_boxes(frame1, frame2) 309 - 310 - def _compute_ssim_boxes( 311 - self, 312 - frame1: np.ndarray, 313 - frame2: np.ndarray, 314 - block_size: int = 160, 315 - ssim_threshold: float = 0.90, 316 - margin: int = 5, 317 - downsample_factor: int = 4, 318 - ) -> List[dict]: 319 - """ 320 - Compare two grayscale frames using block-based SSIM. 321 - 322 - Downsamples before SSIM for speed, computes full SSIM map once, 323 - then pools to blocks. Much faster than 200+ per-block SSIM calls. 224 + bytes 225 + Image as PNG bytes 324 226 """ 325 - from math import ceil 326 - 327 - from skimage.metrics import structural_similarity as ssim 328 - 329 - # Store original dimensions for final boxing 330 - H_orig, W_orig = frame1.shape 331 - 332 - # 1) Downsample by factor (e.g., 4x) for faster SSIM 333 - small1 = frame1[::downsample_factor, ::downsample_factor] 334 - small2 = frame2[::downsample_factor, ::downsample_factor] 335 - 336 - # 2) Convert to float32 in [0, 1] to avoid float64 upcasting 337 - small1 = small1.astype(np.float32) / 255.0 338 - small2 = small2.astype(np.float32) / 255.0 339 - 340 - # 3) Compute SSIM map once on downsampled images 341 - _, ssim_map = ssim( 342 - small1, 343 - small2, 344 - data_range=1.0, # float32 in [0, 1] 345 - full=True, 346 - gaussian_weights=False, # uniform window, matches previous behavior 347 - use_sample_covariance=False, # speed boost 348 - channel_axis=None, # grayscale 349 - ) 350 - 351 - H, W = ssim_map.shape 352 - # Block size in downsampled space 353 - block_size_down = block_size // downsample_factor 354 - rows = ceil(H / block_size_down) 355 - cols = ceil(W / block_size_down) 356 - 357 - # 4) Pad to block grid for clean vectorized pooling 358 - pad_h = rows * block_size_down - H 359 - pad_w = cols * block_size_down - W 360 - if pad_h or pad_w: 361 - ssim_map = np.pad(ssim_map, ((0, pad_h), (0, pad_w)), mode="edge") 362 - 363 - # 5) Vectorized block mean pooling 364 - block_means = ssim_map.reshape( 365 - rows, block_size_down, cols, block_size_down 366 - ).mean(axis=(1, 3)) 367 - changed = (block_means < ssim_threshold).tolist() 368 - 369 - # 6) Reuse shared grouping and boxing logic from utils (uses original dimensions) 370 - from observe.utils import _blocks_to_boxes, _group_changed_blocks 371 - 372 - groups = _group_changed_blocks(changed, rows, cols) 373 - return _blocks_to_boxes(groups, block_size, W_orig, H_orig, margin) 227 + buf = io.BytesIO() 228 + img.save(buf, format="PNG", compress_level=1) 229 + return buf.getvalue() 374 230 375 231 def _user_contents(self, prompt: str, image, entities: bool = False) -> list: 376 232 """Build contents list with optional entity context.""" ··· 419 275 use_prompt : str 420 276 Prompt template filename to use (default: describe_json.txt) 421 277 max_concurrent : int 422 - Maximum number of concurrent API requests (default: 5) 278 + Maximum number of concurrent API requests (default: 10) 423 279 output_path : Optional[Path] 424 280 Path to write JSONL output (when None, no output file is written) 425 281 """ ··· 467 323 468 324 # Create vision requests for all qualified frames 469 325 for frame_data in qualified_frames: 470 - # Load crop image from bytes - keep it open until request completes 471 - crop_img = Image.open(io.BytesIO(frame_data["crop_bytes"])) 326 + # Load frame image from bytes - keep it open until request completes 327 + frame_img = Image.open(io.BytesIO(frame_data["frame_bytes"])) 472 328 473 329 req = batch.create( 474 330 contents=self._user_contents( 475 331 "Analyze this screenshot frame from a screencast recording.", 476 - crop_img, 332 + frame_img, 477 333 ), 478 334 model=GEMINI_LITE, 479 335 system_instruction=system_instruction, ··· 486 342 # Attach metadata for tracking (store bytes, not PIL images) 487 343 req.frame_id = frame_data["frame_id"] 488 344 req.timestamp = frame_data["timestamp"] 489 - req.box_2d = frame_data["box_2d"] 490 345 req.retry_count = 0 491 - req.crop_bytes = frame_data["crop_bytes"] # Store bytes for reuse 346 + req.frame_bytes = frame_data["frame_bytes"] # Store bytes for reuse 492 347 req.request_type = RequestType.DESCRIBE_JSON 493 348 req.json_analysis = None # Will store the JSON analysis result 494 349 req.meeting_analysis = None # Will store meeting analysis if applicable 495 350 req.requests = [] # Track all requests for this frame 496 - req.initial_image = crop_img # Keep reference to close after completion 351 + req.initial_image = frame_img # Keep reference to close after completion 497 352 498 353 batch.add(req) 499 354 ··· 569 424 # Check for meeting analysis 570 425 if visible_category == "meeting": 571 426 logger.info(f"Frame {req.frame_id}: Triggering meeting analysis") 572 - # Load full frame on-demand for meeting analysis 573 - full_image = self._load_full_frame(req.frame_id) 427 + # Reload frame image from cached bytes (already full frame) 428 + meeting_img = Image.open(io.BytesIO(req.frame_bytes)) 574 429 575 430 batch.update( 576 431 req, 577 432 contents=self._user_contents( 578 433 "Analyze this meeting screenshot.", 579 - full_image, 434 + meeting_img, 580 435 entities=True, 581 436 ), 582 437 model=GEMINI_LITE, ··· 587 442 ) 588 443 # Don't close yet - batch needs it for encoding 589 444 # Store reference for cleanup later 590 - req.meeting_image = full_image 445 + req.meeting_image = meeting_img 591 446 592 447 # Close initial image since DESCRIBE_JSON is complete 593 448 if hasattr(req, "initial_image") and req.initial_image: ··· 604 459 logger.info( 605 460 f"Frame {req.frame_id}: Triggering text extraction for category '{visible_category}'" 606 461 ) 607 - # Load crop image from cached bytes 608 - crop_img = Image.open(io.BytesIO(req.crop_bytes)) 462 + # Reload frame image from cached bytes 463 + text_img = Image.open(io.BytesIO(req.frame_bytes)) 609 464 610 465 # Update request for text extraction and re-add 611 466 batch.update( 612 467 req, 613 468 contents=self._user_contents( 614 469 "Extract text from this screenshot frame.", 615 - crop_img, 470 + text_img, 616 471 entities=True, 617 472 ), 618 473 model=GEMINI_LITE, ··· 623 478 ) 624 479 # Don't close yet - batch needs it for encoding 625 480 # Store reference for cleanup later 626 - req.text_image = crop_img 481 + req.text_image = text_img 627 482 628 483 # Close initial image since DESCRIBE_JSON is complete 629 484 if hasattr(req, "initial_image") and req.initial_image: ··· 638 493 result = { 639 494 "frame_id": req.frame_id, 640 495 "timestamp": req.timestamp, 641 - "box_2d": req.box_2d, 642 496 "requests": req.requests, 643 497 } 644 498 ··· 678 532 req.text_image = None 679 533 680 534 # Aggressively clear heavy fields now that request is finalized 681 - req.crop_bytes = None 535 + req.frame_bytes = None 682 536 req.json_analysis = None 683 537 req.meeting_analysis = None 684 538 ··· 729 583 { 730 584 "frame_id": frame["frame_id"], 731 585 "timestamp": frame["timestamp"], 732 - "box_2d": frame["box_2d"], 733 586 } 734 587 for frame in qualified_frames 735 588 ],

+1 -1

observe/describe_json.txt

··· 1 1 # Screenshot Context Analysis 2 2 3 - You are analyzing a desktop screenshot cropped to focus on a specific area. 3 + You are analyzing a desktop screenshot from a screencast recording. 4 4 5 5 Respond with JSON describing the context: 6 6

+1 -1

observe/describe_text.txt

··· 1 1 # Screenshot Text Extraction 2 2 3 - You are extracting text from a desktop screenshot cropped to focus on a specific area. 3 + You are extracting text from a desktop screenshot. 4 4 5 5 Extract all visible text from the image and format it as markdown. 6 6

-56

observe/utils.py

··· 266 266 if header: 267 267 return [header] + frames 268 268 return frames 269 - 270 - 271 - def _group_changed_blocks(changed, grid_rows, grid_cols): 272 - """Group contiguous changed blocks using iterative DFS.""" 273 - groups = [] 274 - visited = [[False] * grid_cols for _ in range(grid_rows)] 275 - 276 - def dfs(i, j, group): 277 - stack = [(i, j)] 278 - while stack: 279 - ci, cj = stack.pop() 280 - if ci < 0 or ci >= grid_rows or cj < 0 or cj >= grid_cols: 281 - continue 282 - if visited[ci][cj] or not changed[ci][cj]: 283 - continue 284 - visited[ci][cj] = True 285 - group.append((ci, cj)) 286 - for ni, nj in [(ci - 1, cj), (ci + 1, cj), (ci, cj - 1), (ci, cj + 1)]: 287 - stack.append((ni, nj)) 288 - 289 - for i in range(grid_rows): 290 - for j in range(grid_cols): 291 - if changed[i][j] and not visited[i][j]: 292 - group = [] 293 - dfs(i, j, group) 294 - groups.append(group) 295 - 296 - return groups 297 - 298 - 299 - def _blocks_to_boxes(groups, block_size, width, height, margin): 300 - """Convert groups of changed blocks to bounding boxes.""" 301 - boxes = [] 302 - for group in groups: 303 - # Calculate bounding box in pixel coordinates 304 - min_x = width 305 - min_y = height 306 - max_x = 0 307 - max_y = 0 308 - for i, j in group: 309 - x0 = j * block_size 310 - y0 = i * block_size 311 - x1 = min(x0 + block_size, width) 312 - y1 = min(y0 + block_size, height) 313 - min_x = min(min_x, x0) 314 - min_y = min(min_y, y0) 315 - max_x = max(max_x, x1) 316 - max_y = max(max_y, y1) 317 - # Add margin 318 - min_x = max(0, min_x - margin) 319 - min_y = max(0, min_y - margin) 320 - max_x = min(width, max_x + margin) 321 - max_y = min(height, max_y + margin) 322 - # Format as [y_min, x_min, y_max, x_max] 323 - boxes.append({"box_2d": [min_y, min_x, max_y, max_x]}) 324 - return boxes

-1

pyproject.toml

··· 69 69 "torch", 70 70 "scipy", 71 71 "opencv-python", 72 - "scikit-image", 73 72 "PyGObject", 74 73 "watchdog", 75 74 "claude-agent-sdk>=0.1.0",

-17

tests/conftest.py

··· 235 235 google_mod.genai = genai_mod 236 236 sys.modules["google"] = google_mod 237 237 sys.modules["google.genai"] = genai_mod 238 - if "skimage.metrics" not in sys.modules: 239 - metrics_mod = types.ModuleType("skimage.metrics") 240 - 241 - def structural_similarity(a, b, full=False): 242 - a = np.asarray(a, dtype=float) 243 - b = np.asarray(b, dtype=float) 244 - diff = np.mean(np.abs(a - b)) / 255.0 245 - score = 1.0 - diff 246 - if full: 247 - return score, None 248 - return score 249 - 250 - metrics_mod.structural_similarity = structural_similarity 251 - skimage_mod = types.ModuleType("skimage") 252 - skimage_mod.metrics = metrics_mod 253 - sys.modules["skimage"] = skimage_mod 254 - sys.modules["skimage.metrics"] = metrics_mod 255 238 if "cv2" not in sys.modules: 256 239 cv2_mod = types.ModuleType("cv2") 257 240 cv2_mod.COLOR_RGB2LAB = 0

Configure Feed

Configure Feed