phase 3

2026-03-23 14:42:36 -03:00
parent 71fd0510de
commit 5ed876d694
17 changed files with 767 additions and 137 deletions
--- a/tests/detect/manual/push_logs.py
+++ b/tests/detect/manual/push_logs.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""
+Push a stream of log events to Redis for UI testing.
+
+Usage:
+    python tests/detect/manual/push_logs.py [--job JOB_ID] [--port PORT] [--count N] [--delay SECS]
+
+Opens: http://mpr.local.ar/detection/?job=<JOB_ID>
+"""
+
+import argparse
+import json
+import random
+import time
+from datetime import datetime, timezone
+
+import redis
+
+STAGES = ["FrameExtractor", "SceneFilter", "YOLODetector", "OCRStage", "BrandResolver", "VLMLocal", "Aggregator"]
+LEVELS = ["INFO", "INFO", "INFO", "INFO", "WARNING", "DEBUG", "ERROR"]  # weighted toward INFO
+MESSAGES = {
+    "FrameExtractor": [
+        "Starting extraction: sample.mp4 (120.0s, 1920x1080, fps=2)",
+        "Extracted 240 frames",
+        "Frame extraction complete",
+    ],
+    "SceneFilter": [
+        "Filtering duplicate scenes (hamming_threshold=8)",
+        "Removed 180 duplicate frames",
+        "Kept 60 unique frames (75% reduction)",
+    ],
+    "YOLODetector": [
+        "Loading yolov8n.pt (fp16)",
+        "Processing batch 1/3 (20 frames)",
+        "Processing batch 2/3 (20 frames)",
+        "Processing batch 3/3 (20 frames)",
+        "Detected 45 regions across 60 frames",
+    ],
+    "OCRStage": [
+        "Running PaddleOCR on 45 regions",
+        "Extracted text from 32 regions",
+        "Resolved 28 brands via OCR",
+    ],
+    "BrandResolver": [
+        "Matching against brand dictionary (12 brands)",
+        "Exact matches: 20, Fuzzy matches: 8",
+        "Unresolved: 4 regions → escalating to VLM",
+    ],
+    "VLMLocal": [
+        "Loading moondream2 (int4, 2.1GB VRAM)",
+        "Processing 4 unresolved crops",
+        "Resolved 3/4 crops, 1 → cloud escalation",
+    ],
+    "Aggregator": [
+        "Compiling detection report",
+        "Found 6 unique brands, 31 total appearances",
+        "Report complete",
+    ],
+}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--job", default="manual-test")
+    parser.add_argument("--port", type=int, default=6382)
+    parser.add_argument("--count", type=int, default=50)
+    parser.add_argument("--delay", type=float, default=0.2)
+    args = parser.parse_args()
+
+    r = redis.Redis(port=args.port, decode_responses=True)
+    key = f"detect_events:{args.job}"
+
+    print(f"Pushing {args.count} log events to {key} (redis port {args.port})")
+    print(f"Open: http://mpr.local.ar/detection/?job={args.job}")
+    print()
+
+    for i in range(args.count):
+        stage = random.choice(STAGES)
+        level = random.choice(LEVELS)
+        msg = random.choice(MESSAGES[stage])
+
+        event = {
+            "event": "log",
+            "level": level,
+            "stage": stage,
+            "msg": f"[{i+1}/{args.count}] {msg}",
+            "ts": datetime.now(timezone.utc).isoformat(),
+        }
+
+        r.rpush(key, json.dumps(event))
+        print(f"  {level:7s} {stage:16s} {msg[:60]}")
+        time.sleep(args.delay)
+
+    print(f"\nDone. {args.count} events pushed.")
+
+
+if __name__ == "__main__":
+    main()
--- a/tests/detect/manual/push_pipeline.py
+++ b/tests/detect/manual/push_pipeline.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+"""
+Simulate a full pipeline run — pushes all event types in sequence.
+
+Usage:
+    python tests/detect/manual/push_pipeline.py [--job JOB_ID] [--port PORT] [--delay SECS]
+
+Opens: http://mpr.local.ar/detection/?job=<JOB_ID>
+"""
+
+import argparse
+import json
+import time
+from datetime import datetime, timezone
+
+import redis
+
+
+def ts():
+    return datetime.now(timezone.utc).isoformat()
+
+
+def push(r, key, event):
+    event["ts"] = event.get("ts", ts())
+    r.rpush(key, json.dumps(event))
+    etype = event["event"]
+    detail = event.get("msg", event.get("stage", ""))
+    print(f"  [{etype:14s}] {detail}")
+    return event
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--job", default="pipeline-test")
+    parser.add_argument("--port", type=int, default=6382)
+    parser.add_argument("--delay", type=float, default=0.5)
+    args = parser.parse_args()
+
+    r = redis.Redis(port=args.port, decode_responses=True)
+    key = f"detect_events:{args.job}"
+
+    # Clear previous events for this job
+    r.delete(key)
+
+    print(f"Simulating pipeline run → {key}")
+    print(f"Open: http://mpr.local.ar/detection/?job={args.job}")
+    print()
+
+    delay = args.delay
+
+    # Stage 1: Frame extraction
+    push(r, key, {"event": "log", "level": "INFO", "stage": "FrameExtractor",
+                   "msg": "Starting extraction: soccer_clip.mp4 (60.0s, 1920x1080, fps=2)"})
+    time.sleep(delay)
+
+    push(r, key, {"event": "stats_update",
+                   "frames_extracted": 120, "frames_after_scene_filter": 0,
+                   "regions_detected": 0, "regions_resolved_by_ocr": 0,
+                   "regions_escalated_to_local_vlm": 0, "regions_escalated_to_cloud_llm": 0,
+                   "cloud_llm_calls": 0, "processing_time_seconds": 3.2, "estimated_cloud_cost_usd": 0})
+    time.sleep(delay)
+
+    push(r, key, {"event": "log", "level": "INFO", "stage": "FrameExtractor",
+                   "msg": "Extracted 120 frames"})
+    time.sleep(delay)
+
+    # Stage 2: Scene filter
+    push(r, key, {"event": "log", "level": "INFO", "stage": "SceneFilter",
+                   "msg": "Filtering duplicates (hamming_threshold=8)"})
+    time.sleep(delay)
+
+    push(r, key, {"event": "stats_update",
+                   "frames_extracted": 120, "frames_after_scene_filter": 45,
+                   "regions_detected": 0, "regions_resolved_by_ocr": 0,
+                   "regions_escalated_to_local_vlm": 0, "regions_escalated_to_cloud_llm": 0,
+                   "cloud_llm_calls": 0, "processing_time_seconds": 5.1, "estimated_cloud_cost_usd": 0})
+    time.sleep(delay)
+
+    push(r, key, {"event": "log", "level": "INFO", "stage": "SceneFilter",
+                   "msg": "Kept 45 frames (62.5% reduction)"})
+    time.sleep(delay)
+
+    # Stage 3: YOLO detection
+    push(r, key, {"event": "log", "level": "INFO", "stage": "YOLODetector",
+                   "msg": "Loading yolov8n.pt (fp16, 1.2GB VRAM)"})
+    time.sleep(delay)
+
+    for batch in range(1, 4):
+        push(r, key, {"event": "log", "level": "DEBUG", "stage": "YOLODetector",
+                       "msg": f"Processing batch {batch}/3 (15 frames)"})
+        time.sleep(delay * 0.5)
+
+    push(r, key, {"event": "stats_update",
+                   "frames_extracted": 120, "frames_after_scene_filter": 45,
+                   "regions_detected": 32, "regions_resolved_by_ocr": 0,
+                   "regions_escalated_to_local_vlm": 0, "regions_escalated_to_cloud_llm": 0,
+                   "cloud_llm_calls": 0, "processing_time_seconds": 12.4, "estimated_cloud_cost_usd": 0})
+    time.sleep(delay)
+
+    # Stage 4: OCR
+    push(r, key, {"event": "log", "level": "INFO", "stage": "OCRStage",
+                   "msg": "Running PaddleOCR on 32 regions"})
+    time.sleep(delay)
+
+    push(r, key, {"event": "stats_update",
+                   "frames_extracted": 120, "frames_after_scene_filter": 45,
+                   "regions_detected": 32, "regions_resolved_by_ocr": 24,
+                   "regions_escalated_to_local_vlm": 0, "regions_escalated_to_cloud_llm": 0,
+                   "cloud_llm_calls": 0, "processing_time_seconds": 18.7, "estimated_cloud_cost_usd": 0})
+    time.sleep(delay)
+
+    # Stage 5: Brand resolver
+    push(r, key, {"event": "log", "level": "INFO", "stage": "BrandResolver",
+                   "msg": "Matched 20 exact, 4 fuzzy. 8 unresolved → VLM"})
+    time.sleep(delay)
+
+    # Emit some detections
+    for brand, conf in [("Nike", 0.95), ("Emirates", 0.91), ("Adidas", 0.88), ("Coca-Cola", 0.82)]:
+        push(r, key, {"event": "detection",
+                       "brand": brand, "confidence": conf, "source": "ocr",
+                       "timestamp": 12.5, "duration": 0.5, "content_type": "soccer_broadcast",
+                       "frame_ref": 25})
+        time.sleep(delay * 0.3)
+
+    # Stage 6: VLM escalation
+    push(r, key, {"event": "log", "level": "INFO", "stage": "VLMLocal",
+                   "msg": "Processing 8 unresolved crops with moondream2"})
+    time.sleep(delay)
+
+    push(r, key, {"event": "log", "level": "WARNING", "stage": "VLMLocal",
+                   "msg": "Low confidence on 2 crops, escalating to cloud LLM"})
+    time.sleep(delay)
+
+    push(r, key, {"event": "stats_update",
+                   "frames_extracted": 120, "frames_after_scene_filter": 45,
+                   "regions_detected": 32, "regions_resolved_by_ocr": 24,
+                   "regions_escalated_to_local_vlm": 8, "regions_escalated_to_cloud_llm": 2,
+                   "cloud_llm_calls": 2, "processing_time_seconds": 28.3, "estimated_cloud_cost_usd": 0.0042})
+    time.sleep(delay)
+
+    # More detections from VLM
+    push(r, key, {"event": "detection",
+                   "brand": "Mastercard", "confidence": 0.76, "source": "local_vlm",
+                   "timestamp": 34.0, "duration": 1.0, "content_type": "soccer_broadcast",
+                   "frame_ref": 68})
+    time.sleep(delay * 0.3)
+
+    push(r, key, {"event": "detection",
+                   "brand": "Heineken", "confidence": 0.71, "source": "cloud_llm",
+                   "timestamp": 45.5, "duration": 0.5, "content_type": "soccer_broadcast",
+                   "frame_ref": 91})
+    time.sleep(delay * 0.3)
+
+    # Final
+    push(r, key, {"event": "log", "level": "INFO", "stage": "Aggregator",
+                   "msg": "Report complete: 6 brands, 26 total appearances"})
+    time.sleep(delay)
+
+    push(r, key, {"event": "job_complete", "job_id": args.job,
+                   "report": {
+                       "video_source": "soccer_clip.mp4",
+                       "content_type": "soccer_broadcast",
+                       "duration_seconds": 60.0,
+                       "brands": {
+                           "Nike": {"total_appearances": 8, "total_screen_time": 4.0, "avg_confidence": 0.93},
+                           "Emirates": {"total_appearances": 6, "total_screen_time": 3.0, "avg_confidence": 0.89},
+                           "Adidas": {"total_appearances": 5, "total_screen_time": 2.5, "avg_confidence": 0.85},
+                           "Coca-Cola": {"total_appearances": 4, "total_screen_time": 2.0, "avg_confidence": 0.80},
+                           "Mastercard": {"total_appearances": 2, "total_screen_time": 1.0, "avg_confidence": 0.76},
+                           "Heineken": {"total_appearances": 1, "total_screen_time": 0.5, "avg_confidence": 0.71},
+                       },
+                   }})
+
+    print(f"\nPipeline simulation complete.")
+
+
+if __name__ == "__main__":
+    main()