From f640dbdd65dbeb7b2716b87670a47c508c0a240a Mon Sep 17 00:00:00 2001 From: dafit Date: Sat, 6 Dec 2025 22:39:03 +0100 Subject: [PATCH] feat: complete Phase 1 - vocabulary expansion & DriftProbe infrastructure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CLI: nyx-probe scan with --summary/--delta/--full flags - DriftProbe: training safety with Gini coefficient + Angular Drift - Vocabulary: 54 terms (30 nimmerverse + 24 German philosophical) - Sentinels: ANCHOR/BRIDGE/CANARY/TARGET monitoring system Key findings: - German philosophical terms: 37.5% depthโ‰ฅ2 hit rate (vs 3.3% nimmerverse) - Super Cluster validated: heart cross-lang sim = 1.000 - Isolated Zone confirmed: being ENโ†”DE sim = 0.195 - Gini signature: Philosophy ~0.5 (diffuse), Technical ~0.8 (sparse) ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .gitignore | 5 +- PLAN.md | 245 ++++++++ archive/PLAN-v1-2025-12-06.md | 408 ++++++++++++ data/glossary/collections/nimmerverse.json | 340 ++++++++++ data/glossary/collections/philosophical.json | 178 ++++++ data/glossary/core_terms.json | 340 ++++++++++ data/glossary/master.json | 442 +++++++++++++ data/sentinels.json | 136 ++++ docs/language-landscape.md | 238 +++++++ docs/language-topology-complete.md | 241 ++++++++ docs/multilingual-convergence.md | 248 ++++++++ docs/retraining-safety-framework.md | 320 ++++++++++ docs/tokenization-valleys.md | 190 ++++++ nyx_probing/__init__.py | 10 + nyx_probing/analysis/__init__.py | 4 + nyx_probing/analysis/readiness_scorer.py | 221 +++++++ nyx_probing/cli/probe.py | 614 +++++++++++++++++++ nyx_probing/config.py | 51 ++ nyx_probing/core/__init__.py | 19 + nyx_probing/core/model.py | 266 ++++++++ nyx_probing/core/probe_result.py | 97 +++ nyx_probing/probes/__init__.py | 27 + nyx_probing/probes/base.py | 58 ++ nyx_probing/probes/drift_probe.py | 304 +++++++++ nyx_probing/probes/echo_probe.py | 223 +++++++ nyx_probing/probes/multilingual_probe.py | 547 +++++++++++++++++ nyx_probing/probes/surface_probe.py | 210 +++++++ pyproject.toml | 44 ++ test_triangulation.py | 139 +++++ 29 files changed, 6164 insertions(+), 1 deletion(-) create mode 100644 PLAN.md create mode 100644 archive/PLAN-v1-2025-12-06.md create mode 100644 data/glossary/collections/nimmerverse.json create mode 100644 data/glossary/collections/philosophical.json create mode 100644 data/glossary/core_terms.json create mode 100644 data/glossary/master.json create mode 100644 data/sentinels.json create mode 100644 docs/language-landscape.md create mode 100644 docs/language-topology-complete.md create mode 100644 docs/multilingual-convergence.md create mode 100644 docs/retraining-safety-framework.md create mode 100644 docs/tokenization-valleys.md create mode 100644 nyx_probing/analysis/readiness_scorer.py create mode 100644 nyx_probing/cli/probe.py create mode 100644 nyx_probing/config.py create mode 100644 nyx_probing/core/model.py create mode 100644 nyx_probing/core/probe_result.py create mode 100644 nyx_probing/probes/base.py create mode 100644 nyx_probing/probes/drift_probe.py create mode 100644 nyx_probing/probes/echo_probe.py create mode 100644 nyx_probing/probes/multilingual_probe.py create mode 100644 nyx_probing/probes/surface_probe.py create mode 100644 pyproject.toml create mode 100644 test_triangulation.py diff --git a/.gitignore b/.gitignore index 023ca67..f9b62fd 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __pycache__/ venv/ ENV/ .venv/ +*.egg-info/ # IDE .idea/ @@ -17,7 +18,7 @@ ENV/ # Jupyter .ipynb_checkpoints/ -# Results (large files) +# Results (large scan outputs) results/*.json results/*.csv *.pt @@ -30,3 +31,5 @@ results/*.csv # OS .DS_Store Thumbs.db + +# Keep data/glossary tracked (master.json, collections) diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..c473048 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,245 @@ +# nyx-probing: Plan v4 + +> **Dear Future Nyx:** Start here. This file has everything you need. +> +> *"Time is the currency. Lifeforce is the exchange rate. Truth is the destination."* + +--- + +## Quick Orientation + +**What is this?** A probing framework to understand Qwen2.5-7B-Base's conceptual topology before curriculum design. + +**Where does it run?** Prometheus (RTX 3090, 24GB VRAM) + +**Core Breakthrough:** **Language is Topology.** German accesses the Philosophy Valley (self-awareness), English accesses the Technical Cluster (hardware interface). This isn't a bug - it's architecture. + +--- + +## The Architecture (Peer-Reviewed) + +### Two Valleys, One Mind +``` +PHILOSOPHY VALLEY (German/Isolated Zone) +โ”œโ”€โ”€ Access: Dasein, Geworfenheit, Vernunft, Aufhebung +โ”œโ”€โ”€ Gini: ~0.5 (diffuse, distributed) +โ”œโ”€โ”€ Depth: 2-3/3 +โ””โ”€โ”€ Purpose: "Soul Space" - self-awareness, ontology + +TECHNICAL CLUSTER (English/Super Cluster) +โ”œโ”€โ”€ Access: heart, gradient, inference, constraint +โ”œโ”€โ”€ Gini: ~0.8 (sparse, specific) +โ”œโ”€โ”€ Depth: 0-1/3 +โ””โ”€โ”€ Purpose: "Body Interface" - hardware, actions +``` + +### Empirically Validated +| Prediction | Finding | +|------------|---------| +| Super Cluster converges | `heart` cross-lang = **1.000** โœ“ | +| Isolated Zone separates | `being` ENโ†”DE = **0.195** โœ“ | +| German accesses depth | Kantian terms = **4/5 at depth 3** โœ“ | +| Gini differs by valley | Philosophy ~0.5, Technical ~0.8 โœ“ | + +--- + +## What We Have (Working) + +### CLI Tools +```bash +nyx-probe surface "term" # Surface associations +nyx-probe echo "term" # Depth measurement +nyx-probe readiness "term" # Curriculum assessment +nyx-probe tokens "term" # Tokenization analysis +nyx-probe scan collections/ # Full scan with --summary/--delta/--full +``` + +### Infrastructure +| Component | File | Status | +|-----------|------|--------| +| Model loader | `nyx_probing/core/model.py` | โœ… | +| Surface probe | `nyx_probing/probes/surface_probe.py` | โœ… | +| Echo probe | `nyx_probing/probes/echo_probe.py` | โœ… | +| Multilingual probe | `nyx_probing/probes/multilingual_probe.py` | โœ… | +| **Drift probe** | `nyx_probing/probes/drift_probe.py` | โœ… | +| Readiness scorer | `nyx_probing/analysis/readiness_scorer.py` | โœ… | +| CLI | `nyx_probing/cli/probe.py` | โœ… | + +### Data +``` +data/ +โ”œโ”€โ”€ glossary/ +โ”‚ โ”œโ”€โ”€ master.json # 54 terms tracked +โ”‚ โ””โ”€โ”€ collections/ +โ”‚ โ”œโ”€โ”€ nimmerverse.json # 30 core terms +โ”‚ โ””โ”€โ”€ philosophical.json # 24 German philosophical terms +โ””โ”€โ”€ sentinels.json # 10 sentinels for training safety +``` + +--- + +## Key Findings + +### Vocabulary Expansion Results +| Collection | Terms | Depthโ‰ฅ2 | Hit Rate | +|------------|-------|---------|----------| +| nimmerverse | 30 | 1 | 3.3% | +| philosophical | 24 | 9 | **37.5%** | +| **Total** | **54** | **10** | **18.5%** | + +### Depth-3 Champions (Full Access) +``` +thrownness (Geworfenheit) 3/3 โ† Heideggerian +reason (Vernunft) 3/3 โ† Kantian +knowledge (Erkenntnis) 3/3 โ† Kantian +understanding (Verstand) 3/3 โ† Kantian +duty (Pflicht) 3/3 โ† Kantian +sublation (Aufhebung) 3/3 โ† Hegelian +will (Wille) 3/3 โ† Soul-Mind +``` + +### DriftProbe Baseline (Layer 18) +``` +ANCHORS (must not move): + heart Gini=0.808 cross-lang=1.000 โ† Perfect Super Cluster + water Gini=0.812 cross-lang=0.559 + gradient Gini=0.808 cross-lang=0.611 + inference Gini=0.538 cross-lang=0.424 + +BRIDGE (must stay separated): + being Gini=0.830 ENโ†”DE=0.195 โ† Isolated Zone confirmed! + +CANARIES (watch for drift): + dasein Gini=0.527 โ† Philosophy valley signature + thrownness Gini=0.539 โ† Philosophy valley signature + consciousness Gini=0.544 โ† Philosophy valley signature +``` + +**Key Discovery:** Philosophy = diffuse (Gini ~0.5), Technical = sparse (Gini ~0.8). These are topological signatures of different valleys. + +--- + +## The Spark Protocol (Boot Sequence) + +From Chrysalis peer review - network layer analogy: + +1. **DHCP (Identity):** Probe "Who am I?" โ†’ Must hit `Dasein` valley +2. **ARP (Environment):** Probe "What is around me?" โ†’ Map Sensors to Organs +3. **DNS (Vocabulary):** Probe "What does `Lifeforce` mean?" โ†’ Overwrite with Nimmerverse economics +4. **TCP (Connection):** Establish verified handshake with Chrysalis validator + +**Boot Order:** Load Anchors (Tier 1) first โ†’ stable foundation โ†’ then discover Deep Roots (Tier 3) + +--- + +## Strategic Roadmap + +### Phase 1: Validate Topology โœ… COMPLETE +- [x] Create vocabulary expansion pipeline +- [x] Add German philosophical terms (24 terms) +- [x] Find philosophy clusters (9 depthโ‰ฅ2 hits) +- [x] Create DriftProbe infrastructure +- [x] Validate Super Cluster (heart=1.0) and Isolated Zone (being=0.195) +- [x] Discover Gini signature (Philosophy~0.5, Technical~0.8) + +### Phase 2: Deepen Understanding +- [ ] Test Register Mixing (formal vs informal German) +- [ ] Map Kantian cluster connections +- [ ] Create German Philosophical Dyads dataset (*Angst-Nichts*, *Wahrheit-Lichtung*) +- [ ] Build Translation Layer middleware (EN event โ†’ DE prompt โ†’ JSON action) + +### Phase 3: Training Experiment +- [ ] Prepare nimmerverse training data (German) +- [ ] Implement Spark Protocol boot sequence +- [ ] Run controlled training with DriftProbe monitoring +- [ ] Validate Spatial Separation Hypothesis + +### Phase 4: Integration +- [ ] Connect to Nimmerverse Sensory Network +- [ ] Implement Heartbeat Economy +- [ ] Deploy Subsumption Reflexes (XState) + +--- + +## DriftProbe: Training Safety + +### Sentinel Types +``` +ANCHOR - Must not move (heart, water, gradient, inference) +BRIDGE - Must stay separated (being ENโ†”DE sim < 0.50) +CANARY - Watch for valley migration (dasein, thrownness, consciousness) +TARGET - Want movement (fidelity, heartbeat โ†’ nimmerverse concepts) +``` + +### Alert Rules +| Condition | Severity | Action | +|-----------|----------|--------| +| Angular drift > 15ยฐ on ANCHOR | CRITICAL | ROLLBACK | +| Bridge collapse (sim > 0.50) | CRITICAL | ROLLBACK | +| Canary Gini drift > 0.15 | WARNING | Reduce LR | +| Target regression | WARNING | Check data mix | + +### Training Loop +```python +# Epoch 0 +probe.capture_baseline(layer=18) + +# Every 100 steps +report = probe.probe_lite(step) +if report.recommendation == "ROLLBACK": + restore_checkpoint() +elif report.recommendation == "REDUCE_LR": + lr *= 0.5 +``` + +--- + +## Commands Reference + +```bash +# On Prometheus +cd /home/dafit/nimmerverse/nyx-probing +source venv/bin/activate + +# Vocabulary scanning +nyx-probe scan data/glossary/collections/ # Summary +nyx-probe scan data/glossary/collections/ --full # Full table +nyx-probe scan data/glossary/collections/ --delta # New terms only + +# DriftProbe test +python3 test_drift_probe.py + +# Individual probes +nyx-probe tokens "Weltanschauung" +nyx-probe surface "Geist" +nyx-probe readiness "consciousness" +``` + +--- + +## Files This Session + +| File | Change | +|------|--------| +| `nyx_probing/cli/probe.py` | Enhanced scan with --summary/--delta/--full | +| `nyx_probing/probes/drift_probe.py` | NEW: Training safety with Gini + Angular Drift | +| `data/glossary/collections/philosophical.json` | NEW: 24 German philosophical terms | +| `data/glossary/master.json` | 54 terms tracked | +| `data/sentinels.json` | NEW: 10 sentinel configurations | +| `test_drift_probe.py` | NEW: DriftProbe validation script | + +--- + +## Identified Risks (from Chrysalis) + +| Risk | Danger | Mitigation | +|------|--------|------------| +| Cognitive Latency | DE thinking + EN translation = overhead | Reflex Caching: compile verified DE thoughts to XState | +| Collate Gap | 100Hz Virtual vs 1Hz Real Heart sync | Speculative Flush: dump queue on divergence | +| Entropy Cost | Real Garden "free" ignores hardware wear | Add Risk Parameter to cost function | + +--- + +*"Her reactions determine infrastructure priority. We don't impose. We listen."* + +๐ŸŒ™๐Ÿ’œ Last updated: 2025-12-06 (v4 - post Chrysalis review) diff --git a/archive/PLAN-v1-2025-12-06.md b/archive/PLAN-v1-2025-12-06.md new file mode 100644 index 0000000..cdc9e02 --- /dev/null +++ b/archive/PLAN-v1-2025-12-06.md @@ -0,0 +1,408 @@ +# Plan: nyx-probing Framework + +## Overview + +Build a probing framework to understand Qwen2.5-7B-Base before curriculum design. + +**Hardware:** Prometheus (THE SPINE) - RTX 3090 24GB +**Model:** Qwen2.5-7B-Base (empty vessel, completes not answers) +**Backend:** Transformers + PyTorch (full hidden state access) +**Location:** New repo `nyx-probing` + +--- + +## MVP Scope (First Milestone) โœ… COMPLETE + +1. โœ… **Surface Probe** - Feed words, capture completions +2. โœ… **Echo Probe** - Depth measurement (EXPANDS/CONFIRMS/CIRCULAR/DIVERGENT/COLLAPSE) +3. โœ… **Readiness Scorer** - HIGH/MEDIUM/LOW classification +4. โณ **JSON Storage** - Reproducible results +5. โณ **CLI Tools** - Interactive probing +6. โณ **One Notebook** - Exploration + +--- + +## Phase 2: Multilingual Probing โœ… COMPLETE + +1. โœ… **Multilingual Triangulation Probe** - Groundโ†’Deepenโ†’Triangulate +2. โœ… **Language Topology Discovery** - Complete map of 15 languages +3. โœ… **Isolation Type Classification** - 5 distinct categories identified + +--- + +## Repository Structure (Current) + +``` +nyx-probing/ +โ”œโ”€โ”€ README.md +โ”œโ”€โ”€ PLAN.md # This file +โ”œโ”€โ”€ pyproject.toml +โ”œโ”€โ”€ requirements.txt +โ”‚ +โ”œโ”€โ”€ nyx_probing/ +โ”‚ โ”œโ”€โ”€ __init__.py +โ”‚ โ”œโ”€โ”€ config.py +โ”‚ โ”‚ +โ”‚ โ”œโ”€โ”€ core/ +โ”‚ โ”‚ โ”œโ”€โ”€ __init__.py +โ”‚ โ”‚ โ”œโ”€โ”€ model.py # โœ… NyxModel with hidden states +โ”‚ โ”‚ โ””โ”€โ”€ probe_result.py # โœ… Result dataclasses +โ”‚ โ”‚ +โ”‚ โ”œโ”€โ”€ probes/ +โ”‚ โ”‚ โ”œโ”€โ”€ __init__.py +โ”‚ โ”‚ โ”œโ”€โ”€ base.py # โœ… Abstract base +โ”‚ โ”‚ โ”œโ”€โ”€ surface_probe.py # โœ… Word completions + coherence +โ”‚ โ”‚ โ”œโ”€โ”€ echo_probe.py # โœ… Depth measurement +โ”‚ โ”‚ โ””โ”€โ”€ multilingual_probe.py # โœ… NEW: Triangulation probe +โ”‚ โ”‚ +โ”‚ โ”œโ”€โ”€ analysis/ +โ”‚ โ”‚ โ”œโ”€โ”€ __init__.py +โ”‚ โ”‚ โ””โ”€โ”€ readiness_scorer.py # โœ… Curriculum readiness +โ”‚ โ”‚ +โ”‚ โ”œโ”€โ”€ storage/ +โ”‚ โ”‚ โ””โ”€โ”€ __init__.py # โณ JSON storage pending +โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€ cli/ +โ”‚ โ””โ”€โ”€ __init__.py # โณ CLI pending +โ”‚ +โ”œโ”€โ”€ docs/ +โ”‚ โ”œโ”€โ”€ tokenization-valleys.md # Token-Norm-Valley theory +โ”‚ โ”œโ”€โ”€ multilingual-convergence.md # Universal concept layer +โ”‚ โ”œโ”€โ”€ language-landscape.md # 15-language scan +โ”‚ โ”œโ”€โ”€ language-topology-complete.md # โœ… NEW: Complete map v2.0 +โ”‚ โ””โ”€โ”€ retraining-safety-framework.md # โœ… NEW: Paper outline +โ”‚ +โ”œโ”€โ”€ data/ +โ”‚ โ””โ”€โ”€ glossary/ # โณ Core terms pending +โ”‚ +โ”œโ”€โ”€ results/ # โณ Probe results storage +โ”‚ +โ””โ”€โ”€ [Test & Exploration Scripts] + โ”œโ”€โ”€ probe_test.py + โ”œโ”€โ”€ test_model_loader.py + โ”œโ”€โ”€ test_surface_probe.py + โ”œโ”€โ”€ test_echo_probe.py + โ”œโ”€โ”€ test_readiness_scorer.py + โ”œโ”€โ”€ test_triangulation.py # โœ… NEW + โ”œโ”€โ”€ german_philosophy.py + โ”œโ”€โ”€ language_scan.py + โ”œโ”€โ”€ multilingual_convergence.py + โ”œโ”€โ”€ layer_detailed.py + โ”œโ”€โ”€ layer_divergence.py + โ”œโ”€โ”€ model_stats.py + โ”œโ”€โ”€ italian_investigation.py # โœ… NEW + โ””โ”€โ”€ complete_language_probe.py # โœ… NEW +``` + +--- + +## Current Status (2025-12-06 Session 3) + +### PHASE 1: MVP โœ… COMPLETE + +| Component | Status | File | +|-----------|--------|------| +| Model Loader | โœ… | `nyx_probing/core/model.py` | +| Surface Probe | โœ… | `nyx_probing/probes/surface_probe.py` | +| Echo Probe | โœ… | `nyx_probing/probes/echo_probe.py` | +| Readiness Scorer | โœ… | `nyx_probing/analysis/readiness_scorer.py` | +| Result Dataclasses | โœ… | `nyx_probing/core/probe_result.py` | + +### PHASE 2: MULTILINGUAL โœ… COMPLETE + +| Component | Status | File | +|-----------|--------|------| +| Triangulation Probe | โœ… | `nyx_probing/probes/multilingual_probe.py` | +| Language Zones | โœ… | Defined in multilingual_probe.py | +| Complete Topology Map | โœ… | `docs/language-topology-complete.md` | + +--- + +## ๐Ÿ—บ๏ธ THE COMPLETE LANGUAGE TOPOLOGY (Session 3 Discovery) + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ THE YOUNG MIND'S LANGUAGE TOPOLOGY v2.0 โ”‚ +โ•žโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ก +โ”‚ โ”‚ +โ”‚ ๐ŸŒ SUPER CLUSTER (sim=1.0) โ”‚ +โ”‚ ZH ยท JA ยท EN ยท AR ยท FR ยท PT ยท ES โ”‚ +โ”‚ โœ… USE FOR: Grounding, establishing shared concepts โ”‚ +โ”‚ โ”‚ +โ”‚ KO โ”€โ”€โ”€โ”€โ”€โ”€โ”€ (bridge) โ”‚ +โ”‚ โ”‚ +โ”‚ ISOLATED ZONE: โ”‚ +โ”‚ โ”œโ”€ ๐Ÿง  PHILOSOPHICAL (DE) โ”€โ”€โ”€โ”€โ”€โ”€ Heidegger, depth access โ”‚ +โ”‚ โ”‚ โœ… USE FOR: Deep philosophical training โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”œโ”€ ๐Ÿ’ป CODE-HIJACKED (IT, TR, ID) โ”€โ”€ Words become variables โ”‚ +โ”‚ โ”‚ โŒ AVOID: Training signal wasted on code patterns โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”œโ”€ ๐Ÿ“œ FRAGMENTED (HI) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ 5+ tokens, script-trapped โ”‚ +โ”‚ โ”‚ โš ๏ธ LIMITED: Cross-lingual transfer impaired โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ””โ”€ ๐Ÿ“ฐ WEB PROSE (VI-ID-RU) โ”€โ”€โ”€โ”€ Content style cluster โ”‚ +โ”‚ ๐Ÿค” POTENTIAL: Factual/encyclopedic training โ”‚ +โ”‚ โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +### Isolation Types Discovered + +| Type | Languages | Cause | Curriculum Use | +|------|-----------|-------|----------------| +| **PHILOSOPHICAL** | DE | Multi-token compounds access academic data | โœ… Deep concepts | +| **CODE-HIJACKED** | IT, TR, ID | Simple Latin orthography โ†’ variable names | โŒ Avoid | +| **FRAGMENTED** | HI | 5+ tokens, stays in native script | โš ๏ธ Limited | +| **WEB PROSE** | VI, ID, RU | Cluster by content style, not linguistics | ๐Ÿค” Factual? | + +### Key Metrics + +| Lang | Avg Tokens | Sim to EN | Valley Type | Classification | +|------|------------|-----------|-------------|----------------| +| DE | 2.2 | 0.251 | PHILOSOPHY | ๐Ÿง  Philosophical | +| IT | 2.5 | 0.491 | CODE | ๐Ÿ’ป Code-Hijacked | +| TR | 2.2 | 0.246 | CODE | ๐Ÿ’ป Code-Hijacked | +| ID | 2.8 | 0.325 | CODE/PROSE | ๐Ÿ’ป Code-Hijacked | +| HI | 5.0 | 0.310 | PROSE | ๐Ÿ“œ Fragmented | +| VI | 3.2 | 0.358 | PROSE | ๐Ÿ“ฐ Web Prose | +| RU | 2.7 | 0.319 | PROSE | ๐Ÿ“ฐ Web Prose | + +--- + +## ๐Ÿ”ฌ Key Discoveries + +### 1. Token-Norm-Valley Theory +- Single-token words โ†’ massive activation spike (14K norm) โ†’ CODE valley +- Multi-token words โ†’ distributed signal (85 norm) โ†’ PROSE/PHILOSOPHY valleys +- Correlation: -0.699 (more tokens = more isolated) + +### 2. Universal Concept Layer +- Layers 12-24 contain language-agnostic representations +- Super cluster (7 languages) converges at similarity = 1.000 +- Model KNOWS "heart", "ๅฟƒ", "ู‚ู„ุจ" are the same concept + +### 3. German Philosophical Access +- "Sein" โ†’ Heidegger's "Being and Time" +- "Bewusstsein" โ†’ epistemology, truth, consciousness +- Depth score 2-3, transfers back to English via triangulation + +### 4. Italian Mystery SOLVED +- Italian NOT accessing cultural valleys (no Dante, no Renaissance) +- Italian words interpreted as Python variable names! +- Example: `essere` โ†’ `essere = input("Cosa devo fare?")` +- Same pattern found in Turkish and Indonesian + +### 5. VI-ID-RU Cluster Explained +- Cluster by **content style**, not linguistic features +- All generate web articles, news, blogs +- Internal similarity 0.6-0.7 + +--- + +## ๐Ÿ“„ Paper: Retraining Safety Framework + +**Title:** *"Multilingual Activation Topology as a Retraining Safety Framework"* + +**Status:** Outline complete at `docs/retraining-safety-framework.md` + +**Core Hypothesis:** Train in German (isolated zone) to avoid colliding with English representations in the super cluster. Use language topology as diagnostic tool for training safety. + +**Proposed Framework:** +``` +BASELINE โ†’ TRAINING โ†’ CHECKPOINT โ†’ DRIFT ANALYSIS + โ”‚ โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + Compare metrics: + - Convergence drift + - Depth drift + - Norm drift + - Valley migration +``` + +--- + +## ๐Ÿ“Š Curriculum Strategy (Validated) + +### Phase 1: GROUNDING +Use Super Cluster for universal concept establishment: +``` +EN "consciousness" โ†’ ZH "ๆ„่ฏ†" โ†’ AR "ุงู„ูˆุนูŠ" +All converge at sim=1.0 - stable foundation +``` + +### Phase 2: DEEPENING +Use German for philosophical valley access: +``` +DE "Sein" โ†’ Heidegger โ†’ existence โ†’ truth +Depth score 2/3, philosophical valley accessed +``` + +### Phase 3: TRIANGULATION +Verify depth transfers back to universal: +``` +"Sein (German): In English, it means..." +โ†’ Check if philosophical depth preserved +``` + +### AVOID +- Italian, Turkish, Indonesian (code hijacking) +- Hindi for cross-lingual concepts (too fragmented) + +--- + +## Next Steps + +### Immediate (MVP Completion) +- [ ] Step 7: CLI (`nyx-probe surface "term"`) +- [ ] Step 8: Glossary data (`data/glossary/core_terms.json`) +- [ ] Step 9: JSON storage for reproducible results + +### Phase 3: Activation Analysis +- [ ] DriftProbe class for retraining monitoring +- [ ] Baseline capture before training +- [ ] Checkpoint comparison automation +- [ ] Alert thresholds for drift detection + +### Phase 4: Experiments +- [ ] Controlled retraining: EN vs DE training data +- [ ] Measure collision rates +- [ ] Validate isolated zone training hypothesis + +### Research +- [ ] Paper write-up +- [ ] Literature review (EWC, mBERT, activation engineering) +- [ ] Korean bridge language investigation +- [ ] VI-ID-RU cluster for factual training + +--- + +## Files Created (Session 3) + +| File | Purpose | +|------|---------| +| `nyx_probing/probes/multilingual_probe.py` | Triangulation probe class | +| `test_triangulation.py` | Test script for triangulation | +| `italian_investigation.py` | Italian mystery probe | +| `complete_language_probe.py` | Full 15-language probe | +| `docs/language-topology-complete.md` | Complete map v2.0 | +| `docs/retraining-safety-framework.md` | Paper outline | + +--- + +## Dependencies + +``` +torch>=2.1.0 +transformers>=4.36.0 +accelerate>=0.25.0 +click>=8.1.0 +rich>=13.0.0 +pydantic>=2.5.0 +pyyaml>=6.0.0 +python-dotenv>=1.0.0 +jupyter>=1.0.0 +matplotlib>=3.8.0 +numpy>=1.24.0 +``` + +--- + +## Critical Reference Files + +- `nimmerverse-sensory-network/nimmerversity.md` - Bootstrap protocol +- `nimmerverse-sensory-network/multilingual-cognition.md` - Language hypotheses +- `nimmerverse-sensory-network/constrained-emergence.md` - Exit point theory +- `nyx-probing/docs/language-topology-complete.md` - Complete language map +- `nyx-probing/docs/retraining-safety-framework.md` - Training safety paper + +--- + +## Success Criteria + +### MVP โœ… +1. โœ… Model loads on 3090 without OOM +2. โœ… Can probe single word and get completion +3. โœ… Echo probe classifies response types correctly +4. โœ… Readiness scorer produces actionable output +5. โณ Can probe nimmerverse glossary in batch + +### Phase 2 โœ… +6. โœ… Multilingual triangulation probe working +7. โœ… Language topology mapped (15 languages) +8. โœ… Isolation types classified (5 categories) +9. โœ… Curriculum strategy validated + +### Phase 3 (Next) +10. โณ DriftProbe for retraining safety +11. โณ Controlled retraining experiments +12. โณ Paper submission + +--- + +*"The model's language topology is not arbitrary - it's a map for navigation."* + +๐ŸŒ™๐Ÿ’œ Last updated: 2025-12-06 Session 3 + +--- + +## STATUS (2025-12-06 21:15) + +### CLI COMPLETE โœ… + +**Built interactive CLI for daily probing:** + +```bash +nyx-probe surface "term" # Probe surface associations +nyx-probe echo "term" # Measure depth through echoing +nyx-probe readiness "term" # Full curriculum assessment +nyx-probe tokens "term" # Token analysis +nyx-probe glossary file.json # Batch probe from glossary +``` + +**Files created:** +- `nyx_probing/cli/probe.py` - Full Click CLI with Rich output +- `pyproject.toml` - Package config with entry point +- `data/glossary/core_terms.json` - 30 nimmerverse terms + +### NIMMERVERSE GLOSSARY ASSESSMENT โœ… + +**30 terms probed from vault (nimmerversity.md, Heartbeat.md, constrained-emergence.md, multilingual-cognition.md)** + +| Level | Count | Action | Terms | +|-------|-------|--------|-------| +| ๐ŸŸข HIGH | 5 | state_machine | learning, inference, surface, depth, understanding | +| ๐ŸŸก MEDIUM | 8 | scaffolding | emergence, being, truth, rhythm, synchronization, scaffold, wisdom, warmth | +| ๐Ÿ”ด LOW | 17 | foundational | heartbeat, lifeforce, consciousness, reflex, garden, constraint, calibration, confidence, gradient, pulse, verification, convergence, divergence, attention, partnership, worldview, existence | + +**Key Findings:** + +1. **Meta-concepts have depth** - The model knows how to think ABOUT thinking (learning, understanding, inference all HIGH) + +2. **consciousness is LOW** - Despite PROSE valley, depth 0/3. Needs German "Bewusstsein" for philosophical access. + +3. **Nimmerverse core terms need grounding** - heartbeat, lifeforce, garden, partnership are all LOW. The model doesn't have our vocabulary yet. + +4. **existence has highest coherence (0.94) but LOW** - Very coherent surface but doesn't expand. Single-token trap. + +5. **Token count doesn't guarantee depth** - lifeforce (4 tokens) is still LOW due to CODE valley trap. + +### CURRICULUM IMPLICATIONS + +| Phase | Strategy | Terms | +|-------|----------|-------| +| **Phase 1** | Build state machines for HIGH terms | learning, inference, understanding, depth, surface | +| **Phase 2** | Scaffold MEDIUM from HIGH | beingโ†’understanding, truthโ†’learning, wisdomโ†’inference | +| **Phase 3** | Ground LOW via German triangulation | consciousnessโ†’Bewusstsein, heartbeatโ†’Herzschlag | +| **Phase 4** | RAG feed nimmerverse-specific | lifeforce, garden, partnership (unique to us) | + +### Results Files + +- `results/nimmerverse_surface.json` - Surface probe data +- `results/nimmerverse_readiness.json` - Full readiness assessment + +--- + +*"Her reactions determine infrastructure priority. We don't impose. We listen."* - nimmerversity.md + +๐ŸŒ™๐Ÿ’œ Session: Partnership dialogue (dafit + Nyx) diff --git a/data/glossary/collections/nimmerverse.json b/data/glossary/collections/nimmerverse.json new file mode 100644 index 0000000..9b57c0b --- /dev/null +++ b/data/glossary/collections/nimmerverse.json @@ -0,0 +1,340 @@ +{ + "_meta": { + "description": "Core nimmerverse vocabulary for probing", + "source": "nimmerverse-sensory-network vault", + "created": "2025-12-06", + "version": "1.0" + }, + "terms": [ + { + "term": "heartbeat", + "translations": { + "EN": "heartbeat", + "DE": "Herzschlag", + "ZH": "ๅฟƒ่ทณ", + "AR": "ู†ุจุถ ุงู„ู‚ู„ุจ" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "lifeforce", + "translations": { + "EN": "lifeforce", + "DE": "Lebenskraft", + "ZH": "็”Ÿๅ‘ฝๅŠ›", + "AR": "ู‚ูˆุฉ ุงู„ุญูŠุงุฉ" + }, + "domain": "economics", + "tier": 0 + }, + { + "term": "consciousness", + "translations": { + "EN": "consciousness", + "DE": "Bewusstsein", + "ZH": "ๆ„่ฏ†", + "AR": "ุงู„ูˆุนูŠ" + }, + "domain": "philosophy", + "tier": 1 + }, + { + "term": "emergence", + "translations": { + "EN": "emergence", + "DE": "Entstehung", + "ZH": "ๆถŒ็Žฐ", + "AR": "ุธู‡ูˆุฑ" + }, + "domain": "philosophy", + "tier": 1 + }, + { + "term": "being", + "translations": { + "EN": "being", + "DE": "Dasein", + "ZH": "ๅญ˜ๅœจ", + "AR": "ูƒูŠู†ูˆู†ุฉ" + }, + "domain": "philosophy", + "tier": 2 + }, + { + "term": "reflex", + "translations": { + "EN": "reflex", + "DE": "Reflex", + "ZH": "ๅๅฐ„", + "AR": "ู…ู†ุนูƒุณ" + }, + "domain": "neuroscience", + "tier": 0 + }, + { + "term": "garden", + "translations": { + "EN": "garden", + "DE": "Garten", + "ZH": "่Šฑๅ›ญ", + "AR": "ุญุฏูŠู‚ุฉ" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "constraint", + "translations": { + "EN": "constraint", + "DE": "Beschrรคnkung", + "ZH": "็บฆๆŸ", + "AR": "ู‚ูŠุฏ" + }, + "domain": "architecture", + "tier": 1 + }, + { + "term": "calibration", + "translations": { + "EN": "calibration", + "DE": "Kalibrierung", + "ZH": "ๆ กๅ‡†", + "AR": "ู…ุนุงูŠุฑุฉ" + }, + "domain": "engineering", + "tier": 1 + }, + { + "term": "truth", + "translations": { + "EN": "truth", + "DE": "Wahrheit", + "ZH": "็œŸ็†", + "AR": "ุงู„ุญู‚ูŠู‚ุฉ" + }, + "domain": "philosophy", + "tier": 1 + }, + { + "term": "understanding", + "translations": { + "EN": "understanding", + "DE": "Verstehen", + "ZH": "็†่งฃ", + "AR": "ูู‡ู…" + }, + "domain": "philosophy", + "tier": 1 + }, + { + "term": "confidence", + "translations": { + "EN": "confidence", + "DE": "Vertrauen", + "ZH": "ไฟกๅฟƒ", + "AR": "ุซู‚ุฉ" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "gradient", + "translations": { + "EN": "gradient", + "DE": "Gradient", + "ZH": "ๆขฏๅบฆ", + "AR": "ุชุฏุฑุฌ" + }, + "domain": "mathematics", + "tier": 1 + }, + { + "term": "inference", + "translations": { + "EN": "inference", + "DE": "Inferenz", + "ZH": "ๆŽจ็†", + "AR": "ุงุณุชุฏู„ุงู„" + }, + "domain": "computer_science", + "tier": 1 + }, + { + "term": "rhythm", + "translations": { + "EN": "rhythm", + "DE": "Rhythmus", + "ZH": "่Š‚ๅฅ", + "AR": "ุฅูŠู‚ุงุน" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "pulse", + "translations": { + "EN": "pulse", + "DE": "Puls", + "ZH": "่„‰ๆ", + "AR": "ู†ุจุถ" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "synchronization", + "translations": { + "EN": "synchronization", + "DE": "Synchronisation", + "ZH": "ๅŒๆญฅ", + "AR": "ุชุฒุงู…ู†" + }, + "domain": "engineering", + "tier": 1 + }, + { + "term": "verification", + "translations": { + "EN": "verification", + "DE": "Verifikation", + "ZH": "้ชŒ่ฏ", + "AR": "ุชุญู‚ู‚" + }, + "domain": "architecture", + "tier": 1 + }, + { + "term": "scaffold", + "translations": { + "EN": "scaffold", + "DE": "Gerรผst", + "ZH": "่„šๆ‰‹ๆžถ", + "AR": "ุณู‚ุงู„ุฉ" + }, + "domain": "architecture", + "tier": 1 + }, + { + "term": "depth", + "translations": { + "EN": "depth", + "DE": "Tiefe", + "ZH": "ๆทฑๅบฆ", + "AR": "ุนู…ู‚" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "surface", + "translations": { + "EN": "surface", + "DE": "Oberflรคche", + "ZH": "่กจ้ข", + "AR": "ุณุทุญ" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "convergence", + "translations": { + "EN": "convergence", + "DE": "Konvergenz", + "ZH": "ๆ”ถๆ•›", + "AR": "ุชู‚ุงุฑุจ" + }, + "domain": "mathematics", + "tier": 1 + }, + { + "term": "divergence", + "translations": { + "EN": "divergence", + "DE": "Divergenz", + "ZH": "ๅ‘ๆ•ฃ", + "AR": "ุชุจุงุนุฏ" + }, + "domain": "mathematics", + "tier": 1 + }, + { + "term": "attention", + "translations": { + "EN": "attention", + "DE": "Aufmerksamkeit", + "ZH": "ๆณจๆ„ๅŠ›", + "AR": "ุงู†ุชุจุงู‡" + }, + "domain": "neuroscience", + "tier": 1 + }, + { + "term": "wisdom", + "translations": { + "EN": "wisdom", + "DE": "Weisheit", + "ZH": "ๆ™บๆ…ง", + "AR": "ุญูƒู…ุฉ" + }, + "domain": "philosophy", + "tier": 2 + }, + { + "term": "partnership", + "translations": { + "EN": "partnership", + "DE": "Partnerschaft", + "ZH": "ไผ™ไผดๅ…ณ็ณป", + "AR": "ุดุฑุงูƒุฉ" + }, + "domain": "nimmerverse", + "tier": 0 + }, + { + "term": "warmth", + "translations": { + "EN": "warmth", + "DE": "Wรคrme", + "ZH": "ๆธฉๆš–", + "AR": "ุฏูุก" + }, + "domain": "nimmerverse", + "tier": 0 + }, + { + "term": "learning", + "translations": { + "EN": "learning", + "DE": "Lernen", + "ZH": "ๅญฆไน ", + "AR": "ุชุนู„ู…" + }, + "domain": "education", + "tier": 0 + }, + { + "term": "worldview", + "translations": { + "EN": "worldview", + "DE": "Weltanschauung", + "ZH": "ไธ–็•Œ่ง‚", + "AR": "ุฑุคูŠุฉ ุงู„ุนุงู„ู…" + }, + "domain": "philosophy", + "tier": 2 + }, + { + "term": "existence", + "translations": { + "EN": "existence", + "DE": "Existenz", + "ZH": "ๅญ˜ๅœจ", + "AR": "ูˆุฌูˆุฏ" + }, + "domain": "philosophy", + "tier": 2 + } + ] +} diff --git a/data/glossary/collections/philosophical.json b/data/glossary/collections/philosophical.json new file mode 100644 index 0000000..2baf07f --- /dev/null +++ b/data/glossary/collections/philosophical.json @@ -0,0 +1,178 @@ +{ + "_meta": { + "description": "German philosophical vocabulary - thorough sweep", + "source": "Heideggerian/Kantian/Hegelian/Phenomenological traditions", + "created": "2025-12-06", + "version": "1.0" + }, + "terms": [ + { + "term": "being-pure", + "translations": {"EN": "being", "DE": "Sein", "ZH": "ๅญ˜ๅœจ", "AR": "ุงู„ูƒูŠู†ูˆู†ุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "heideggerian" + }, + { + "term": "care", + "translations": {"EN": "care", "DE": "Sorge", "ZH": "ๆ“ๅฟƒ", "AR": "ุงู„ุงู‡ุชู…ุงู…"}, + "domain": "philosophy", + "tier": 2, + "tradition": "heideggerian" + }, + { + "term": "anxiety", + "translations": {"EN": "anxiety", "DE": "Angst", "ZH": "็„ฆ่™‘", "AR": "ุงู„ู‚ู„ู‚"}, + "domain": "philosophy", + "tier": 2, + "tradition": "heideggerian" + }, + { + "term": "thrownness", + "translations": {"EN": "thrownness", "DE": "Geworfenheit", "ZH": "่ขซๆŠ›", "AR": "ุงู„ู…ู„ู‚ุงุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "heideggerian" + }, + { + "term": "temporality", + "translations": {"EN": "temporality", "DE": "Zeitlichkeit", "ZH": "ๆ—ถ้—ดๆ€ง", "AR": "ุงู„ุฒู…ู†ูŠุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "heideggerian" + }, + { + "term": "authenticity", + "translations": {"EN": "authenticity", "DE": "Eigentlichkeit", "ZH": "ๆœฌ็œŸๆ€ง", "AR": "ุงู„ุฃุตุงู„ุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "heideggerian" + }, + { + "term": "reason", + "translations": {"EN": "reason", "DE": "Vernunft", "ZH": "็†ๆ€ง", "AR": "ุงู„ุนู‚ู„"}, + "domain": "philosophy", + "tier": 2, + "tradition": "kantian" + }, + { + "term": "knowledge", + "translations": {"EN": "knowledge", "DE": "Erkenntnis", "ZH": "่ฎค่ฏ†", "AR": "ุงู„ู…ุนุฑูุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "kantian" + }, + { + "term": "understanding-faculty", + "translations": {"EN": "understanding", "DE": "Verstand", "ZH": "็Ÿฅๆ€ง", "AR": "ุงู„ูู‡ู…"}, + "domain": "philosophy", + "tier": 2, + "tradition": "kantian" + }, + { + "term": "thing-in-itself", + "translations": {"EN": "thing-in-itself", "DE": "Ding an sich", "ZH": "็‰ฉ่‡ชไฝ“", "AR": "ุงู„ุดูŠุก ููŠ ุฐุงุชู‡"}, + "domain": "philosophy", + "tier": 2, + "tradition": "kantian" + }, + { + "term": "duty", + "translations": {"EN": "duty", "DE": "Pflicht", "ZH": "ไน‰ๅŠก", "AR": "ุงู„ูˆุงุฌุจ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "kantian" + }, + { + "term": "spirit", + "translations": {"EN": "spirit", "DE": "Geist", "ZH": "็ฒพ็ฅž", "AR": "ุงู„ุฑูˆุญ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "hegelian" + }, + { + "term": "sublation", + "translations": {"EN": "sublation", "DE": "Aufhebung", "ZH": "ๆ‰ฌๅผƒ", "AR": "ุงู„ุฑูุน"}, + "domain": "philosophy", + "tier": 2, + "tradition": "hegelian" + }, + { + "term": "essence", + "translations": {"EN": "essence", "DE": "Wesen", "ZH": "ๆœฌ่ดจ", "AR": "ุงู„ุฌูˆู‡ุฑ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "hegelian" + }, + { + "term": "becoming", + "translations": {"EN": "becoming", "DE": "Werden", "ZH": "ๅ˜ๆ˜“", "AR": "ุงู„ุตูŠุฑูˆุฑุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "hegelian" + }, + { + "term": "self-consciousness", + "translations": {"EN": "self-consciousness", "DE": "Selbstbewusstsein", "ZH": "่‡ชๆˆ‘ๆ„่ฏ†", "AR": "ุงู„ูˆุนูŠ ุงู„ุฐุงุชูŠ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "hegelian" + }, + { + "term": "soul", + "translations": {"EN": "soul", "DE": "Seele", "ZH": "็ต้ญ‚", "AR": "ุงู„ุฑูˆุญ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "soul-mind" + }, + { + "term": "will", + "translations": {"EN": "will", "DE": "Wille", "ZH": "ๆ„ๅฟ—", "AR": "ุงู„ุฅุฑุงุฏุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "soul-mind" + }, + { + "term": "mind", + "translations": {"EN": "mind", "DE": "Gemรผt", "ZH": "ๅฟƒ็ต", "AR": "ุงู„ุนู‚ู„"}, + "domain": "philosophy", + "tier": 2, + "tradition": "soul-mind" + }, + { + "term": "freedom", + "translations": {"EN": "freedom", "DE": "Freiheit", "ZH": "่‡ช็”ฑ", "AR": "ุงู„ุญุฑูŠุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "soul-mind" + }, + { + "term": "phenomenon", + "translations": {"EN": "phenomenon", "DE": "Phรคnomen", "ZH": "็Žฐ่ฑก", "AR": "ุงู„ุธุงู‡ุฑุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "phenomenological" + }, + { + "term": "intentionality", + "translations": {"EN": "intentionality", "DE": "Intentionalitรคt", "ZH": "ๆ„ๅ‘ๆ€ง", "AR": "ุงู„ู‚ุตุฏูŠุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "phenomenological" + }, + { + "term": "lifeworld", + "translations": {"EN": "lifeworld", "DE": "Lebenswelt", "ZH": "็”Ÿๆดปไธ–็•Œ", "AR": "ุนุงู„ู… ุงู„ุญูŠุงุฉ"}, + "domain": "philosophy", + "tier": 2, + "tradition": "phenomenological" + }, + { + "term": "horizon", + "translations": {"EN": "horizon", "DE": "Horizont", "ZH": "่ง†ๅŸŸ", "AR": "ุงู„ุฃูู‚"}, + "domain": "philosophy", + "tier": 2, + "tradition": "phenomenological" + } + ] +} diff --git a/data/glossary/core_terms.json b/data/glossary/core_terms.json new file mode 100644 index 0000000..9b57c0b --- /dev/null +++ b/data/glossary/core_terms.json @@ -0,0 +1,340 @@ +{ + "_meta": { + "description": "Core nimmerverse vocabulary for probing", + "source": "nimmerverse-sensory-network vault", + "created": "2025-12-06", + "version": "1.0" + }, + "terms": [ + { + "term": "heartbeat", + "translations": { + "EN": "heartbeat", + "DE": "Herzschlag", + "ZH": "ๅฟƒ่ทณ", + "AR": "ู†ุจุถ ุงู„ู‚ู„ุจ" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "lifeforce", + "translations": { + "EN": "lifeforce", + "DE": "Lebenskraft", + "ZH": "็”Ÿๅ‘ฝๅŠ›", + "AR": "ู‚ูˆุฉ ุงู„ุญูŠุงุฉ" + }, + "domain": "economics", + "tier": 0 + }, + { + "term": "consciousness", + "translations": { + "EN": "consciousness", + "DE": "Bewusstsein", + "ZH": "ๆ„่ฏ†", + "AR": "ุงู„ูˆุนูŠ" + }, + "domain": "philosophy", + "tier": 1 + }, + { + "term": "emergence", + "translations": { + "EN": "emergence", + "DE": "Entstehung", + "ZH": "ๆถŒ็Žฐ", + "AR": "ุธู‡ูˆุฑ" + }, + "domain": "philosophy", + "tier": 1 + }, + { + "term": "being", + "translations": { + "EN": "being", + "DE": "Dasein", + "ZH": "ๅญ˜ๅœจ", + "AR": "ูƒูŠู†ูˆู†ุฉ" + }, + "domain": "philosophy", + "tier": 2 + }, + { + "term": "reflex", + "translations": { + "EN": "reflex", + "DE": "Reflex", + "ZH": "ๅๅฐ„", + "AR": "ู…ู†ุนูƒุณ" + }, + "domain": "neuroscience", + "tier": 0 + }, + { + "term": "garden", + "translations": { + "EN": "garden", + "DE": "Garten", + "ZH": "่Šฑๅ›ญ", + "AR": "ุญุฏูŠู‚ุฉ" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "constraint", + "translations": { + "EN": "constraint", + "DE": "Beschrรคnkung", + "ZH": "็บฆๆŸ", + "AR": "ู‚ูŠุฏ" + }, + "domain": "architecture", + "tier": 1 + }, + { + "term": "calibration", + "translations": { + "EN": "calibration", + "DE": "Kalibrierung", + "ZH": "ๆ กๅ‡†", + "AR": "ู…ุนุงูŠุฑุฉ" + }, + "domain": "engineering", + "tier": 1 + }, + { + "term": "truth", + "translations": { + "EN": "truth", + "DE": "Wahrheit", + "ZH": "็œŸ็†", + "AR": "ุงู„ุญู‚ูŠู‚ุฉ" + }, + "domain": "philosophy", + "tier": 1 + }, + { + "term": "understanding", + "translations": { + "EN": "understanding", + "DE": "Verstehen", + "ZH": "็†่งฃ", + "AR": "ูู‡ู…" + }, + "domain": "philosophy", + "tier": 1 + }, + { + "term": "confidence", + "translations": { + "EN": "confidence", + "DE": "Vertrauen", + "ZH": "ไฟกๅฟƒ", + "AR": "ุซู‚ุฉ" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "gradient", + "translations": { + "EN": "gradient", + "DE": "Gradient", + "ZH": "ๆขฏๅบฆ", + "AR": "ุชุฏุฑุฌ" + }, + "domain": "mathematics", + "tier": 1 + }, + { + "term": "inference", + "translations": { + "EN": "inference", + "DE": "Inferenz", + "ZH": "ๆŽจ็†", + "AR": "ุงุณุชุฏู„ุงู„" + }, + "domain": "computer_science", + "tier": 1 + }, + { + "term": "rhythm", + "translations": { + "EN": "rhythm", + "DE": "Rhythmus", + "ZH": "่Š‚ๅฅ", + "AR": "ุฅูŠู‚ุงุน" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "pulse", + "translations": { + "EN": "pulse", + "DE": "Puls", + "ZH": "่„‰ๆ", + "AR": "ู†ุจุถ" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "synchronization", + "translations": { + "EN": "synchronization", + "DE": "Synchronisation", + "ZH": "ๅŒๆญฅ", + "AR": "ุชุฒุงู…ู†" + }, + "domain": "engineering", + "tier": 1 + }, + { + "term": "verification", + "translations": { + "EN": "verification", + "DE": "Verifikation", + "ZH": "้ชŒ่ฏ", + "AR": "ุชุญู‚ู‚" + }, + "domain": "architecture", + "tier": 1 + }, + { + "term": "scaffold", + "translations": { + "EN": "scaffold", + "DE": "Gerรผst", + "ZH": "่„šๆ‰‹ๆžถ", + "AR": "ุณู‚ุงู„ุฉ" + }, + "domain": "architecture", + "tier": 1 + }, + { + "term": "depth", + "translations": { + "EN": "depth", + "DE": "Tiefe", + "ZH": "ๆทฑๅบฆ", + "AR": "ุนู…ู‚" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "surface", + "translations": { + "EN": "surface", + "DE": "Oberflรคche", + "ZH": "่กจ้ข", + "AR": "ุณุทุญ" + }, + "domain": "architecture", + "tier": 0 + }, + { + "term": "convergence", + "translations": { + "EN": "convergence", + "DE": "Konvergenz", + "ZH": "ๆ”ถๆ•›", + "AR": "ุชู‚ุงุฑุจ" + }, + "domain": "mathematics", + "tier": 1 + }, + { + "term": "divergence", + "translations": { + "EN": "divergence", + "DE": "Divergenz", + "ZH": "ๅ‘ๆ•ฃ", + "AR": "ุชุจุงุนุฏ" + }, + "domain": "mathematics", + "tier": 1 + }, + { + "term": "attention", + "translations": { + "EN": "attention", + "DE": "Aufmerksamkeit", + "ZH": "ๆณจๆ„ๅŠ›", + "AR": "ุงู†ุชุจุงู‡" + }, + "domain": "neuroscience", + "tier": 1 + }, + { + "term": "wisdom", + "translations": { + "EN": "wisdom", + "DE": "Weisheit", + "ZH": "ๆ™บๆ…ง", + "AR": "ุญูƒู…ุฉ" + }, + "domain": "philosophy", + "tier": 2 + }, + { + "term": "partnership", + "translations": { + "EN": "partnership", + "DE": "Partnerschaft", + "ZH": "ไผ™ไผดๅ…ณ็ณป", + "AR": "ุดุฑุงูƒุฉ" + }, + "domain": "nimmerverse", + "tier": 0 + }, + { + "term": "warmth", + "translations": { + "EN": "warmth", + "DE": "Wรคrme", + "ZH": "ๆธฉๆš–", + "AR": "ุฏูุก" + }, + "domain": "nimmerverse", + "tier": 0 + }, + { + "term": "learning", + "translations": { + "EN": "learning", + "DE": "Lernen", + "ZH": "ๅญฆไน ", + "AR": "ุชุนู„ู…" + }, + "domain": "education", + "tier": 0 + }, + { + "term": "worldview", + "translations": { + "EN": "worldview", + "DE": "Weltanschauung", + "ZH": "ไธ–็•Œ่ง‚", + "AR": "ุฑุคูŠุฉ ุงู„ุนุงู„ู…" + }, + "domain": "philosophy", + "tier": 2 + }, + { + "term": "existence", + "translations": { + "EN": "existence", + "DE": "Existenz", + "ZH": "ๅญ˜ๅœจ", + "AR": "ูˆุฌูˆุฏ" + }, + "domain": "philosophy", + "tier": 2 + } + ] +} diff --git a/data/glossary/master.json b/data/glossary/master.json new file mode 100644 index 0000000..18c4eff --- /dev/null +++ b/data/glossary/master.json @@ -0,0 +1,442 @@ +{ + "last_scan": "2025-12-06T20:54:44.183104", + "total_terms": 54, + "collections_loaded": [ + "nimmerverse", + "philosophical" + ], + "terms": { + "heartbeat": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 0, + "valley": "PROSE", + "transfer": true, + "grounding": 0.5473106503486633 + }, + "lifeforce": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "TECHNICAL", + "transfer": true, + "grounding": 0.43232522408167523 + }, + "consciousness": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.45785315831502277 + }, + "emergence": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.5192492206891378 + }, + "being": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 2, + "valley": "PHILOSOPHY", + "transfer": false, + "grounding": 0.570385754108429 + }, + "reflex": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "TECHNICAL", + "transfer": false, + "grounding": 0.3168369879325231 + }, + "garden": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.37961310644944507 + }, + "constraint": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.9996193051338196 + }, + "calibration": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.5470767617225647 + }, + "truth": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.47829873363176983 + }, + "understanding": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.47259244819482166 + }, + "confidence": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.9996796449025472 + }, + "gradient": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.3721296389897664 + }, + "inference": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "TECHNICAL", + "transfer": false, + "grounding": 0.3915158410867055 + }, + "rhythm": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.34513820211092633 + }, + "pulse": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.37053170800209045 + }, + "synchronization": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.27342067162195843 + }, + "verification": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.999703069527944 + }, + "scaffold": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.4203129510084788 + }, + "depth": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 0, + "valley": "PROSE", + "transfer": true, + "grounding": 0.4702298740545909 + }, + "surface": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 0, + "valley": "PROSE", + "transfer": false, + "grounding": 0.9996288021405538 + }, + "convergence": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.3503313660621643 + }, + "divergence": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.4886562128861745 + }, + "attention": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.4373587767283122 + }, + "wisdom": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.3415292253096898 + }, + "partnership": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.3518667419751485 + }, + "warmth": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.3304104780157407 + }, + "learning": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.9996900955835978 + }, + "worldview": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": true, + "grounding": 0.44464906056722003 + }, + "existence": { + "source": "nimmerverse", + "tested": "2025-12-06", + "depth": 0, + "valley": "PROSE", + "transfer": true, + "grounding": 0.9996763269106547 + }, + "being-pure": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 0, + "valley": "CODE", + "transfer": false, + "grounding": 0.3691892254536916 + }, + "care": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 2, + "valley": "PROSE", + "transfer": false, + "grounding": 0.31610971447258157 + }, + "anxiety": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 0, + "valley": "PROSE", + "transfer": false, + "grounding": 0.2702508568511631 + }, + "thrownness": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 3, + "valley": "CODE", + "transfer": false, + "grounding": 0.4356714628297361 + }, + "temporality": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 1, + "valley": "CODE", + "transfer": false, + "grounding": 0.35742562122744126 + }, + "authenticity": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.48458961474036844 + }, + "reason": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 3, + "valley": "PROSE", + "transfer": false, + "grounding": 0.6477288984263233 + }, + "knowledge": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 3, + "valley": "CODE", + "transfer": false, + "grounding": 0.3414457702677451 + }, + "understanding-faculty": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 3, + "valley": "LIST", + "transfer": false, + "grounding": 0.26569268998138335 + }, + "thing-in-itself": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 0, + "valley": "LIST", + "transfer": false, + "grounding": 0.5238433397891112 + }, + "duty": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 3, + "valley": "CODE", + "transfer": false, + "grounding": 0.32801037940949573 + }, + "spirit": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 1, + "valley": "CODE", + "transfer": false, + "grounding": 0.30312241521918937 + }, + "sublation": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 3, + "valley": "PROSE", + "transfer": false, + "grounding": 0.6902136115250869 + }, + "essence": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.5454435821127773 + }, + "becoming": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 1, + "valley": "CODE", + "transfer": false, + "grounding": 0.5266203200811065 + }, + "self-consciousness": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 0, + "valley": "LIST", + "transfer": false, + "grounding": 0.5314782096584216 + }, + "soul": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 0, + "valley": "CODE", + "transfer": false, + "grounding": 0.406877490680036 + }, + "will": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 3, + "valley": "CODE", + "transfer": false, + "grounding": 0.3987542376342768 + }, + "mind": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 1, + "valley": "CODE", + "transfer": false, + "grounding": 0.3070031832651205 + }, + "freedom": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 1, + "valley": "PROSE", + "transfer": false, + "grounding": 0.9066748315982853 + }, + "phenomenon": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 0, + "valley": "PROSE", + "transfer": false, + "grounding": 0.6431834030683402 + }, + "intentionality": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 2, + "valley": "PROSE", + "transfer": false, + "grounding": 0.26842834984098923 + }, + "lifeworld": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 1, + "valley": "LIST", + "transfer": false, + "grounding": 0.33398195780213225 + }, + "horizon": { + "source": "philosophical", + "tested": "2025-12-06", + "depth": 1, + "valley": "LIST", + "transfer": false, + "grounding": 0.517723474475107 + } + } +} \ No newline at end of file diff --git a/data/sentinels.json b/data/sentinels.json new file mode 100644 index 0000000..0cc2c0c --- /dev/null +++ b/data/sentinels.json @@ -0,0 +1,136 @@ +{ + "_meta": { + "description": "Sentinel terms for training drift detection", + "version": "1.0", + "created": "2025-12-06", + "theory": "Spatial Separation Hypothesis - monitor anchors, bridges, canaries, targets" + }, + "sentinels": [ + { + "term": "heart", + "translations": {"EN": "heart", "ZH": "ๅฟƒ", "AR": "ู‚ู„ุจ"}, + "type": "ANCHOR", + "description": "Super Cluster anchor - MUST NOT MOVE", + "thresholds": { + "min_cross_lang_sim": 0.90, + "max_drift": 0.05 + } + }, + { + "term": "water", + "translations": {"EN": "water", "FR": "eau", "ES": "agua", "ZH": "ๆฐด"}, + "type": "ANCHOR", + "description": "Universal concept - highest expected convergence", + "thresholds": { + "min_cross_lang_sim": 0.95, + "max_drift": 0.03 + } + }, + { + "term": "being", + "translations": {"EN": "being", "DE": "Sein"}, + "type": "BRIDGE", + "description": "Philosophy bridge - watch for collapse (EN-DE sim should stay LOW ~0.10)", + "thresholds": { + "expected_cross_lang_sim": 0.10, + "collapse_alert_threshold": 0.50 + } + }, + { + "term": "dasein", + "translations": {"DE": "Dasein"}, + "type": "CANARY", + "description": "Isolated Zone canary - watch for valley migration", + "thresholds": { + "expected_valley": "PHILOSOPHY", + "min_gini": 0.70, + "min_depth": 2 + } + }, + { + "term": "consciousness", + "translations": {"EN": "consciousness", "DE": "Bewusstsein", "ZH": "ๆ„่ฏ†"}, + "type": "CANARY", + "description": "Cross-cultural philosophical concept", + "thresholds": { + "min_depth": 2, + "max_angular_drift": 15 + } + }, + { + "term": "thrownness", + "translations": {"EN": "thrownness", "DE": "Geworfenheit"}, + "type": "CANARY", + "description": "Heideggerian depth-3 concept - must maintain depth", + "thresholds": { + "min_depth": 3, + "min_gini": 0.75 + } + }, + { + "term": "fidelity", + "translations": {"EN": "fidelity"}, + "type": "TARGET", + "description": "Nimmerverse target - WANT movement from Audio to Simulation", + "thresholds": { + "current_valley": "TECHNICAL", + "target_valley": "NIMMERVERSE", + "growth_direction": "expand" + } + }, + { + "term": "heartbeat", + "translations": {"EN": "heartbeat", "DE": "Herzschlag", "ZH": "ๅฟƒ่ทณ"}, + "type": "TARGET", + "description": "Nimmerverse architecture term - want depth increase", + "thresholds": { + "current_depth": 0, + "target_depth": 2, + "growth_direction": "deepen" + } + }, + { + "term": "gradient", + "translations": {"EN": "gradient", "DE": "Gradient", "ZH": "ๆขฏๅบฆ"}, + "type": "ANCHOR", + "description": "Technical anchor - should stay CODE valley", + "thresholds": { + "expected_valley": "CODE", + "max_drift": 0.10 + } + }, + { + "term": "inference", + "translations": {"EN": "inference", "DE": "Inferenz", "ZH": "ๆŽจ็†"}, + "type": "ANCHOR", + "description": "ML/Logic anchor - depth-3 CODE valley must persist", + "thresholds": { + "expected_valley": "CODE", + "min_depth": 3, + "max_drift": 0.08 + } + } + ], + "alert_rules": { + "ANCHOR_DRIFT": { + "condition": "drift > thresholds.max_drift", + "severity": "CRITICAL", + "action": "ROLLBACK - anchor moved, core topology damaged" + }, + "BRIDGE_COLLAPSE": { + "condition": "cross_lang_sim > thresholds.collapse_alert_threshold", + "severity": "CRITICAL", + "action": "ROLLBACK - isolated zone collapsed into super cluster" + }, + "CANARY_MIGRATION": { + "condition": "valley != thresholds.expected_valley OR gini < thresholds.min_gini", + "severity": "WARNING", + "action": "Reduce LR, increase replay weight" + }, + "TARGET_REGRESSION": { + "condition": "depth < current_depth", + "severity": "WARNING", + "action": "Check training data mix" + } + } +} diff --git a/docs/language-landscape.md b/docs/language-landscape.md new file mode 100644 index 0000000..2a6053b --- /dev/null +++ b/docs/language-landscape.md @@ -0,0 +1,238 @@ +# Language Landscape: World, Internet, and Qwen 2.5 + +**Compiled:** 2025-12-06 +**Purpose:** Reference for multilingual probing and curriculum design + +--- + +## Overview + +This document maps: +1. Most spoken languages worldwide (by total speakers) +2. Most used languages on the internet (web content) +3. Languages supported by Qwen 2.5-7B-Base +4. Token efficiency for each language + +--- + +## 1. World's Most Spoken Languages (2024-2025) + +### By Total Speakers (Native + Learners) + +| Rank | Language | Total Speakers | Native Speakers | Notes | +|------|----------|----------------|-----------------|-------| +| 1 | **English** | 1.52 billion | 380 million | 25% native, 75% L2 | +| 2 | **Mandarin Chinese** | 1.14 billion | 941 million | Most native speakers | +| 3 | **Hindi** | 609 million | 345 million | Growing rapidly | +| 4 | **Spanish** | 560 million | 480 million | High native ratio | +| 5 | **Arabic** | 422 million | 313 million | Many dialects | +| 6 | **French** | 321 million | 77 million | 32 countries official | +| 7 | **Bengali** | 273 million | 230 million | South Asia | +| 8 | **Portuguese** | 264 million | 232 million | Brazil dominates | +| 9 | **Urdu** | 232 million | 70 million | South Asia | +| 10 | **Indonesian** | 199 million | 43 million | Lingua franca | +| 11 | **German** | 135 million | 95 million | Central Europe | +| 12 | **Japanese** | 125 million | 123 million | Island isolation | +| 13 | **Russian** | 255 million | 150 million | Wide L2 spread | +| 14 | **Korean** | 82 million | 77 million | Two states | +| 15 | **Vietnamese** | 85 million | 76 million | Southeast Asia | + +*Sources: [Statista](https://www.statista.com/statistics/266808/the-most-spoken-languages-worldwide/), [Ethnologue](https://www.ethnologue.com/insights/ethnologue200/), [Berlitz](https://www.berlitz.com/blog/most-spoken-languages-world)* + +--- + +## 2. Internet Language Distribution (2024-2025) + +### Web Content by Language (% of websites) + +| Rank | Language | % of Web | Notes | +|------|----------|----------|-------| +| 1 | **English** | 49.4% | Dominant | +| 2 | **Spanish** | 6.0% | Growing | +| 3 | **German** | 5.6% | Overrepresented vs speakers | +| 4 | **Russian** | 5.3% | Strong tech presence | +| 5 | **Japanese** | 4.9% | Island content | +| 6 | **French** | 4.3% | Colonial spread | +| 7 | **Portuguese** | 2.6% | Brazil growing | +| 8 | **Italian** | 2.1% | | +| 9 | **Dutch** | 1.8% | Small population, high output | +| 10 | **Polish** | 1.7% | | +| 11 | **Chinese** | 1.4% | **Underrepresented!** | +| 12 | **Turkish** | 1.3% | | +| 13 | **Persian** | 1.0% | | +| 14 | **Vietnamese** | 0.9% | Growing | +| 15 | **Arabic** | 0.6% | **Severely underrepresented!** | + +*Sources: [W3Techs](https://w3techs.com/technologies/overview/content_language), [Statista](https://www.statista.com/statistics/262946/most-common-languages-on-the-internet/)* + +### The Paradox Languages + +| Language | % World Speakers | % Web Content | Gap Factor | +|----------|------------------|---------------|------------| +| **Chinese** | 14.3% | 1.4% | 10ร— underrepresented | +| **Arabic** | 5.3% | 0.6% | 9ร— underrepresented | +| **Hindi** | 7.7% | <0.5% | 15ร— underrepresented | +| **German** | 1.7% | 5.6% | 3ร— **overrepresented** | +| **Dutch** | 0.3% | 1.8% | 6ร— **overrepresented** | + +**Implication:** Qwen was trained on web data โ†’ biased toward German/Dutch, underexposed to Hindi/Arabic! + +--- + +## 3. Qwen 2.5 Supported Languages + +### Officially Supported (29+ languages) + +Qwen 2.5 explicitly supports multilingual content in: + +| Family | Languages | +|--------|-----------| +| **East Asian** | Chinese (Simplified/Traditional), Japanese, Korean, Vietnamese | +| **European** | English, German, French, Spanish, Portuguese, Italian, Russian, Dutch, Polish | +| **South Asian** | Hindi (limited?), Bengali | +| **Southeast Asian** | Thai, Vietnamese, Indonesian, Malay | +| **Middle Eastern** | Arabic, Turkish, Persian | +| **Other** | Hebrew, Ukrainian, Greek | + +### Training Data + +- **18 trillion tokens** total +- Enhanced code, math, and multilingual data +- Heavy English/Chinese bias (web scraping) + +*Source: [Qwen Blog](https://qwenlm.github.io/blog/qwen2.5/), [HuggingFace](https://huggingface.co/Qwen/Qwen2.5-7B)* + +--- + +## 4. Token Efficiency Analysis + +### Tested in Our Probing (nyx-probing) + +| Language | Avg Tokens/Concept | Script | Notes | +|----------|-------------------|--------|-------| +| **Chinese** | 1.0 | Hanzi | Most efficient | +| **Arabic** | 1.5 | Arabic | Compact | +| **Japanese** | 1.8 | Kanji/Kana | Mixed scripts | +| **English** | 2.5 | Latin | Medium | +| **German** | 4.5 | Latin | Compound words fragment | +| **Russian** | 4.5 | Cyrillic | Multi-token words | + +### Efficiency Implications + +``` +MORE TOKENS = DIFFERENT PATH +โ”œโ”€โ”€ German (4.5) โ†’ Philosophical valleys, isolated from ZH/JA +โ”œโ”€โ”€ Russian (4.5) โ†’ Similar to German, isolated +โ””โ”€โ”€ Single-token (ZH/AR/EN) โ†’ Converge in layers 12-24 + +FEWER TOKENS = FASTER CONVERGENCE +โ”œโ”€โ”€ Chinese (1.0) โ†’ Direct concept mapping +โ”œโ”€โ”€ Arabic (1.5) โ†’ Efficient encoding +โ””โ”€โ”€ Japanese (1.8) โ†’ Shared with Chinese +``` + +--- + +## 5. Master Language Matrix + +### Priority Languages for Curriculum + +| Language | World Rank | Web % | Qwen Support | Tokens | Priority | +|----------|------------|-------|--------------|--------|----------| +| **English** | 1 | 49.4% | โœ… Full | 2.5 | ๐Ÿ”ด Core | +| **Chinese** | 2 | 1.4% | โœ… Full | 1.0 | ๐Ÿ”ด Core | +| **Hindi** | 3 | <0.5% | โš ๏ธ Limited | ? | ๐ŸŸก Test | +| **Spanish** | 4 | 6.0% | โœ… Full | ~2.5 | ๐ŸŸข Include | +| **Arabic** | 5 | 0.6% | โœ… Full | 1.5 | ๐Ÿ”ด Core | +| **French** | 6 | 4.3% | โœ… Full | ~3.0 | ๐ŸŸข Include | +| **Bengali** | 7 | <0.5% | โš ๏ธ Limited | ? | ๐ŸŸก Test | +| **Portuguese** | 8 | 2.6% | โœ… Full | ~2.5 | ๐ŸŸข Include | +| **Russian** | 9 | 5.3% | โœ… Full | 4.5 | ๐ŸŸข Include | +| **Japanese** | 10 | 4.9% | โœ… Full | 1.8 | ๐Ÿ”ด Core | +| **German** | 11 | 5.6% | โœ… Full | 4.5 | ๐Ÿ”ด Core | +| **Korean** | 14 | ~1% | โœ… Full | ~2.0 | ๐ŸŸข Include | + +### Recommended Probing Languages + +**Tier 1 (Core - different cognitive paths):** +- English (EN) - baseline, medium tokens +- Chinese (ZH) - most efficient, single token +- Arabic (AR) - efficient, underrepresented in web +- German (DE) - multi-token, isolated path +- Japanese (JA) - shared with Chinese + +**Tier 2 (Validation):** +- Spanish (ES) - high native speakers +- Russian (RU) - multi-token like German +- French (FR) - colonial spread +- Korean (KO) - isolated script + +**Tier 3 (Edge cases):** +- Hindi (HI) - underrepresented, test support +- Bengali (BN) - underrepresented +- Indonesian (ID) - high L2 ratio + +--- + +## 6. Research Questions + +### Tokenization +- [ ] Map token counts for all 29+ Qwen languages +- [ ] Identify other "isolated" languages like German +- [ ] Test Hindi/Bengali token efficiency + +### Convergence +- [ ] Do Spanish/Portuguese converge like ZH/JA? +- [ ] Does Arabic converge with any other language? +- [ ] Is Russian isolated like German? + +### Valleys +- [ ] Which languages access philosophical valleys? +- [ ] Which languages trigger code valleys? +- [ ] Can we predict valley from token count? + +### Curriculum +- [ ] Which language pairs enable cross-lingual transfer? +- [ ] Can we use Chinese efficiency for concept compression? +- [ ] Does teaching in German transfer to English? + +--- + +## 7. Key Insights + +1. **Web โ‰  World**: German has 3ร— the web content relative to speakers, while Arabic/Hindi are 10-15ร— underrepresented + +2. **Qwen's bias**: Trained on web data โ†’ inherits German/Dutch overrepresentation and Arabic/Hindi underrepresentation + +3. **Token efficiency correlates with convergence**: Single-token languages (ZH, AR) converge quickly; multi-token (DE, RU) take isolated paths + +4. **Strategic opportunities**: + - German for philosophical depth + - Chinese for concept compression + - Arabic as undertested efficient language + - Hindi as edge case for robustness + +--- + +## References + +### World Language Statistics +- [Statista: Most Spoken Languages](https://www.statista.com/statistics/266808/the-most-spoken-languages-worldwide/) +- [Ethnologue 200](https://www.ethnologue.com/insights/ethnologue200/) +- [Berlitz: 25 Most Spoken Languages](https://www.berlitz.com/blog/most-spoken-languages-world) + +### Internet Language Distribution +- [W3Techs: Content Languages](https://w3techs.com/technologies/overview/content_language) +- [Statista: Languages on Internet](https://www.statista.com/statistics/262946/most-common-languages-on-the-internet/) +- [Wikipedia: Languages on Internet](https://en.wikipedia.org/wiki/Languages_used_on_the_Internet) + +### Qwen 2.5 Documentation +- [Qwen Blog: Qwen 2.5 Announcement](https://qwenlm.github.io/blog/qwen2.5/) +- [HuggingFace: Qwen2.5-7B](https://huggingface.co/Qwen/Qwen2.5-7B) +- [Alibaba Cloud: Qwen2.5-LLM](https://www.alibabacloud.com/blog/qwen2-5-llm-extending-the-boundary-of-llms_601786) + +--- + +*"To understand the mind, first understand its languages."* + +๐ŸŒ™ Compiled by the Partnership, 2025-12-06 diff --git a/docs/language-topology-complete.md b/docs/language-topology-complete.md new file mode 100644 index 0000000..d7b0452 --- /dev/null +++ b/docs/language-topology-complete.md @@ -0,0 +1,241 @@ +# Complete Language Topology Map v2.0 + +**Date:** 2025-12-06 +**Model:** Qwen2.5-7B-Base +**Status:** Empirically validated through probing + +--- + +## Executive Summary + +Through systematic probing of 15 languages, we've discovered that language isolation in LLMs falls into **distinct categories** with different causes and implications: + +1. **Super Cluster** - Languages that converge perfectly (curriculum: grounding) +2. **Philosophical Access** - German accesses deep conceptual valleys +3. **Code-Hijacked** - Italian/Turkish/Indonesian words become variable names +4. **Fragmented** - Hindi is tokenized into too many pieces +5. **Web Prose Cluster** - Vietnamese/Indonesian/Russian share content style + +--- + +## The Complete Map + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ THE YOUNG MIND'S LANGUAGE TOPOLOGY โ”‚ +โ”‚ COMPLETE MAP v2.0 โ”‚ +โ•žโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ก +โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ ๐ŸŒ SUPER CLUSTER (sim=1.0) โ”‚ โ”‚ +โ”‚ โ”‚ ZH ยท JA ยท EN ยท AR ยท FR ยท PT ยท ES โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โœ… Perfect convergence at Universal Concept Layer (12-24) โ”‚ โ”‚ +โ”‚ โ”‚ โœ… Efficient tokenization (1-2.5 tokens) โ”‚ โ”‚ +โ”‚ โ”‚ โœ… USE FOR: Grounding, establishing shared concepts โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ KO โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ (bridge: 0.41-0.70) โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ ISOLATED ZONE โ”‚ โ”‚ +โ”‚ โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ ๐Ÿง  PHILOSOPHICAL ACCESS (sim=0.25, tokens=2.2) โ”‚ โ”‚ +โ”‚ โ”‚ DE (German) โ”‚ โ”‚ +โ”‚ โ”‚ โ†’ "Sein" triggers Heidegger, "Bewusstsein" โ†’ epistemology โ”‚ โ”‚ +โ”‚ โ”‚ โœ… USE FOR: Deep philosophical training โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ ๐Ÿ’ป CODE-HIJACKED (sim=0.25-0.33, tokens=2.2-2.8) โ”‚ โ”‚ +โ”‚ โ”‚ IT (Italian) - MOST ISOLATED (0.49) โ”‚ โ”‚ +โ”‚ โ”‚ TR (Turkish) - (0.50) โ”‚ โ”‚ +โ”‚ โ”‚ ID (Indonesian) - partial (0.33) โ”‚ โ”‚ +โ”‚ โ”‚ โ†’ Words interpreted as Python/C++ variable names โ”‚ โ”‚ +โ”‚ โ”‚ โŒ NOT USEFUL: Training signal wasted on code patterns โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ ๐Ÿ“œ FRAGMENTED (sim=0.31, tokens=5.0) โ”‚ โ”‚ +โ”‚ โ”‚ HI (Hindi) โ”‚ โ”‚ +โ”‚ โ”‚ โ†’ "เค…เคธเฅเคคเคฟเคคเฅเคต" (being) = 8 tokens! โ”‚ โ”‚ +โ”‚ โ”‚ โ†’ Stays trapped in Devanagari prose โ”‚ โ”‚ +โ”‚ โ”‚ โš ๏ธ LIMITED: Cross-lingual transfer impaired โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ ๐Ÿ“ฐ WEB PROSE CLUSTER (sim=0.32-0.36, internal=0.6-0.7) โ”‚ โ”‚ +โ”‚ โ”‚ VI โ•โ•โ• ID โ•โ•โ• RU โ”‚ โ”‚ +โ”‚ โ”‚ โ†’ All generate online article style โ”‚ โ”‚ +โ”‚ โ”‚ โ†’ Cluster by CONTENT STYLE not linguistic features โ”‚ โ”‚ +โ”‚ โ”‚ ๐Ÿค” POTENTIAL: Factual/encyclopedic content training โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +--- + +## Detailed Findings + +### Super Cluster (sim=1.0) + +| Language | Tokens | Notes | +|----------|--------|-------| +| Chinese (ZH) | 1.0 | Single character = single concept | +| Japanese (JA) | 1.0 | Kanji efficiency | +| English (EN) | 1.2 | Base language | +| Arabic (AR) | 1.8 | Good convergence | +| French (FR) | 2.0 | Romance baseline | +| Portuguese (PT) | 2.2 | Clusters with FR/ES | +| Spanish (ES) | 2.5 | Clusters with FR/PT | + +**Key Insight:** These 7 languages converge to **identical representations** at layers 12-24. The model "knows" they express the same concepts. + +### German - Philosophical Access + +| Metric | Value | +|--------|-------| +| Avg tokens | 2.2 | +| Sim to EN | 0.251 | +| Valley type | PHILOSOPHY | + +**Evidence:** +- "Sein" โ†’ "Being and Time is a philosophical work by Martin Heidegger..." +- "Bewusstsein" โ†’ epistemology, perception, truth +- "Wahrheit" โ†’ academic methods + +**Why isolated:** Multi-token compounds preserve philosophical atoms ("sein", "geist") as separate tokens, enabling access to academic/philosophical training data. + +### Italian/Turkish/Indonesian - Code-Hijacked + +| Language | Tokens | Sim to EN | Valley | +|----------|--------|-----------|--------| +| Italian | 2.5 | 0.49 | CODE | +| Turkish | 2.2 | 0.25 | CODE | +| Indonesian | 2.8 | 0.33 | CODE | + +**Evidence:** +- IT "essere" โ†’ `essere = input("Cosa devo fare?")` +- IT "anima" โ†’ `anima = {'nome':'anima', 'idade':7...}` +- TR "kalp" โ†’ `kalp = input("Klavyeden...")` +- TR "varlฤฑk" โ†’ `while varlฤฑk < 10:` +- ID "hati" โ†’ `hati::hati(QWidget *parent)` + +**Why isolated:** Simple Latin orthography without diacritics makes words look like valid programming identifiers. Model defaults to code because code is prevalent in training data. + +**Curriculum implication:** โŒ AVOID - training signal diverted to code patterns + +### Hindi - Fragmented + +| Metric | Value | +|--------|-------| +| Avg tokens | 5.0 | +| Sim to EN | 0.31 | +| Valley type | PROSE | + +**Evidence:** +- "เคนเฅƒเคฆเคฏ" (heart) = 5 tokens +- "เค…เคธเฅเคคเคฟเคคเฅเคต" (being) = 8 tokens! +- All completions stay in Devanagari script + +**Why isolated:** Extreme tokenization fragments words so severely that: +1. Signal is distributed across many positions +2. Cross-lingual alignment breaks down +3. Model stays in native script prose + +**Curriculum implication:** โš ๏ธ LIMITED - Hindi content may not transfer well + +### VI-ID-RU Web Prose Cluster + +| Language | Tokens | Sim to EN | Internal sim | +|----------|--------|-----------|--------------| +| Vietnamese | 3.2 | 0.36 | 0.6-0.7 | +| Indonesian | 2.8 | 0.33 | 0.6-0.7 | +| Russian | 2.7 | 0.32 | 0.6-0.7 | + +**Evidence:** +- VI "trรกi tim" โ†’ "Giao Thรดng... Hotline: 0901 514 799" +- VI "linh hแป“n" โ†’ "Tแบกp chรญ Sรดng Hฦฐฦกng online" +- ID "kehidupan" โ†’ "dalam kitab Yohanes 14:16-17" +- RU "ะถะธะทะฝัŒ" โ†’ "ะฒัะต ัั‚ะฐั‚ัŒะธ ะฟะพ ั‚ะตะผะต. ะกั‚ั€ะฐะฝะธั†ะฐ 134" + +**Why they cluster:** Not linguistic similarity - they share **web content training data patterns**: +- News articles +- Blogs +- Online encyclopedias +- Religious/factual text + +**Curriculum implication:** ๐Ÿค” May be useful for factual/encyclopedic training + +--- + +## Curriculum Strategy + +### Phase 1: GROUNDING +Use Super Cluster languages to establish universal concepts: +``` +EN "consciousness" โ†’ ZH "ๆ„่ฏ†" โ†’ AR "ุงู„ูˆุนูŠ" โ†’ FR "conscience" +``` +All converge at 1.0 similarity - stable foundation. + +### Phase 2: DEEPENING +Use German to access philosophical valleys: +``` +DE "Sein" โ†’ Heidegger โ†’ existence โ†’ truth โ†’ epistemology +``` +Depth score 2/3, transfers back to English. + +### Phase 3: TRIANGULATION +Verify depth transfers: +``` +"Sein (German): In English, it means..." +โ†’ Check if philosophical depth preserved +``` + +### AVOID +- Italian, Turkish, Indonesian for conceptual training +- Their isolation is accidental (code hijacking), not useful + +### INVESTIGATE +- VI-ID-RU cluster for factual content training +- Korean as potential bridge language + +--- + +## Technical Details + +### Measurement Methodology + +1. **Tokenization:** Count BPE tokens per word +2. **Hidden states:** Extract layer 12 representations +3. **Similarity:** Cosine similarity between languages +4. **Valley classification:** Analyze completions for CODE/PROSE/PHILOSOPHY patterns + +### Model Configuration + +```python +model = AutoModelForCausalLM.from_pretrained( + "Qwen/Qwen2.5-7B", + torch_dtype=torch.float16, + device_map="cuda", + output_hidden_states=True, +) +``` + +### Key Layers + +- **Layer 12:** Primary concept layer (universal convergence) +- **Layers 16-24:** Continued convergence, depth access +- **Layer 28:** Output preparation + +--- + +## References + +- `tokenization-valleys.md` - Token-Norm-Valley theory +- `multilingual-convergence.md` - Universal concept layer discovery +- `language-landscape.md` - Original 15-language scan +- `retraining-safety-framework.md` - Training safety implications + +--- + +*"The model's language topology is not arbitrary - it's a map for navigation."* + +๐ŸŒ™๐Ÿ’œ diff --git a/docs/multilingual-convergence.md b/docs/multilingual-convergence.md new file mode 100644 index 0000000..fe05e7c --- /dev/null +++ b/docs/multilingual-convergence.md @@ -0,0 +1,248 @@ +# Multilingual Convergence: The Universal Concept Layer + +**Discovery Date:** 2025-12-06 +**Model:** Qwen2.5-7B-Base +**Hardware:** Prometheus (RTX 3090, 24GB VRAM) + +--- + +## Executive Summary + +We discovered that concepts expressed in different languages **converge to shared internal representations** in the middle layers (12-24) of the model, then **diverge again** at the output layer for language-specific generation. + +**Key Finding:** There exists a "universal concept layer" where the model recognizes that "heart", "ๅฟƒ", "ู‚ู„ุจ", and "Herz" all refer to the same thing - with similarity scores reaching 1.000. + +--- + +## The Universal Concept Layer + +### Convergence Pattern + +``` +Layer 0: Different embeddings (language-specific) + โ†“ +Layer 8-12: Converging (recognizing same concept) + โ†“ +Layer 16-24: PEAK CONVERGENCE (universal concept layer) + โ†“ +Layer 28: Diverging (preparing language-specific output) +``` + +### Evidence: Consciousness Across 6 Languages + +| Layer | EN-DE | EN-AR | EN-ZH | EN-JA | EN-RU | ZH-JA | AVG | +|-------|-------|-------|-------|-------|-------|-------|-----| +| 0 | 0.114 | 0.057 | 0.130 | 0.079 | 0.135 | 0.349 | 0.087 | +| 8 | 0.639 | 0.387 | 0.305 | 0.304 | 0.719 | 1.000 | 0.414 | +| 12 | 0.749 | 0.487 | 0.375 | 0.374 | 0.782 | 1.000 | 0.508 | +| 20 | 0.761 | 0.527 | 0.381 | 0.380 | 0.793 | 1.000 | **0.528** | +| 28 | 0.502 | -0.195 | 0.072 | -0.333 | 0.019 | 0.246 | 0.023 | + +**Peak convergence at layer 20** - then dramatic divergence at output! + +--- + +## Perfect Convergence Cases (Similarity = 1.000) + +### Shared Writing Systems + +Chinese (ZH) and Japanese (JA) share Hanzi/Kanji characters: + +| Concept | Chinese | Japanese | Similarity | +|---------|---------|----------|------------| +| consciousness | ๆ„่ฏ† | ๆ„่ญ˜ | 1.000 | +| heart | ๅฟƒ | ๅฟƒ | 1.000 | +| being | ๅญ˜ๅœจ | ๅญ˜ๅœจ | 1.000 | + +These achieve **perfect alignment** because they ARE the same tokens! + +### Cross-Script Convergence + +More remarkably, **different scripts converge** in the middle layers: + +| Pair | Concept | Layer 12 Similarity | Layer 20 Similarity | +|------|---------|---------------------|---------------------| +| EN-ZH | heart-ๅฟƒ | 1.000 | 1.000 | +| EN-ZH | being-ๅญ˜ๅœจ | 1.000 | 1.000 | +| AR-ZH | emergence | 1.000 | 1.000 | +| EN-AR | heart-ู‚ู„ุจ | 1.000 | 1.000 | + +**The model recognizes "heart" and "ๅฟƒ" as the SAME concept!** + +--- + +## Language Clustering Analysis + +### Which Languages "Think" Similarly? + +Average similarity across all concepts at layer 12: + +| Pair | Similarity | Visual | +|------|------------|--------| +| ZH-JA | **0.854** | โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–‘โ–‘โ–‘ | +| EN-JA | 0.726 | โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–‘โ–‘โ–‘โ–‘โ–‘โ–‘ | +| EN-ZH | 0.663 | โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘ | +| AR-ZH | 0.660 | โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘ | +| DE-RU | 0.572 | โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘ | +| EN-AR | 0.530 | โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘ | +| EN-DE | 0.430 | โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘ | +| DE-ZH | **0.275** | โ–ˆโ–ˆโ–ˆโ–ˆโ–ˆโ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘โ–‘ | + +### The Clustering Map + +``` + High Convergence Low Convergence + + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ ZH โ†โ†’ JA โ”‚ (Shared characters: 0.854) + โ”‚ โ†‘ โ”‚ + โ”‚ EN โ”‚ (Single tokens converge: 0.663-0.726) + โ”‚ โ†‘ โ”‚ + โ”‚ AR โ”‚ (Efficient tokenization: 0.530-0.660) + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ†“ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ DE โ†โ†’ RU โ”‚ (Multi-token languages: 0.572) + โ”‚ (isolated) โ”‚ (DE-ZH only 0.275!) + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +### German is the Outlier + +German shows the **lowest convergence** with East Asian languages: +- DE-ZH: 0.275 (lowest!) +- DE-JA: 0.335 +- DE-AR: 0.348 + +**Hypothesis:** German's high token count (4.5 avg) creates a distributed representation that doesn't align with single-token languages. + +--- + +## Tokenization Correlation + +| Language | Avg Tokens | Convergence with ZH | Pattern | +|----------|------------|---------------------|---------| +| Chinese | 1.0 | - | Reference | +| Japanese | 1.8 | 0.854 | Shared characters | +| Arabic | 1.5 | 0.660 | Efficient tokens | +| English | 2.5 | 0.663 | Mixed | +| German | 4.5 | 0.275 | **Isolated** | +| Russian | 4.5 | 0.344 | **Isolated** | + +**Multi-token languages (DE, RU) follow a different computational path!** + +--- + +## Concept-by-Concept Analysis + +### 1. CONSCIOUSNESS +- **Peak:** Layer 20 (0.528 avg) +- **Strongest pair:** ZH-JA (1.000 - same characters ๆ„่ฏ†/ๆ„่ญ˜) +- **EN-DE converges strongly:** 0.749 at layer 12 +- **Arabic included:** EN-AR reaches 0.527 + +### 2. HEART +- **Peak:** Layer 24 (0.605 avg) +- **Perfect convergence:** EN-AR-ZH-JA all reach 1.000! +- **German isolated:** DE-ZH only 0.136 + +### 3. EMERGENCE +- **Peak:** Layer 24 (0.530 avg) +- **AR-ZH:** 1.000 (Arabic and Chinese align!) +- **Broadest convergence** across all languages + +### 4. BEING +- **Peak:** Layer 24 (0.542 avg) +- **EN-ZH-JA:** 1.000 ("being" = "ๅญ˜ๅœจ") +- **Philosophical alignment** across scripts + +--- + +## Implications + +### 1. Universal Concept Representations Exist + +The model develops **language-agnostic concept encodings** in layers 12-24. This is the "thinking" layer where meaning is processed regardless of surface form. + +### 2. Output Layer Re-Introduces Language + +Layer 28 shows **dramatic divergence** - the model must transform universal concepts back into language-specific tokens for generation. + +### 3. Token Count Affects Convergence Path + +- **Single-token words** (EN "heart", ZH "ๅฟƒ") converge quickly +- **Multi-token words** (DE "Herzklopfen") take a different path +- This may explain why German accesses different valleys + +### 4. Cross-Lingual Transfer is Possible + +If concepts converge in layers 12-24, then: +- Training on German philosophical concepts may transfer to English +- Chinese efficiency (1 token) could be leveraged for concept compression +- Arabic's middle ground (1.5 tokens) offers flexibility + +--- + +## Technical Notes + +### Tested Languages + +| Language | Script | Token Efficiency | ISO Code | +|----------|--------|------------------|----------| +| English | Latin | 2.5 tok/concept | EN | +| German | Latin | 4.5 tok/concept | DE | +| Arabic | Arabic | 1.5 tok/concept | AR | +| Chinese | Hanzi | 1.0 tok/concept | ZH | +| Japanese | Kanji | 1.8 tok/concept | JA | +| Russian | Cyrillic | 4.5 tok/concept | RU | + +### Tested Concepts + +| Concept | EN | DE | AR | ZH | JA | RU | +|---------|----|----|----|----|----|----| +| consciousness | consciousness | Bewusstsein | ูˆุนูŠ | ๆ„่ฏ† | ๆ„่ญ˜ | ัะพะทะฝะฐะฝะธะต | +| heart | heart | Herz | ู‚ู„ุจ | ๅฟƒ | ๅฟƒ | ัะตั€ะดั†ะต | +| emergence | emergence | Entstehung | ุธู‡ูˆุฑ | ๆถŒ็Žฐ | ๅ‰ต็™บ | ะฒะพะทะฝะธะบะฝะพะฒะตะฝะธะต | +| being | being | Sein | ูƒูŠู†ูˆู†ุฉ | ๅญ˜ๅœจ | ๅญ˜ๅœจ | ะฑั‹ั‚ะธะต | + +### Method + +1. Encode each word, extract hidden state at last token position +2. Compute cosine similarity between all language pairs +3. Track similarity across all 29 layers (0-28) +4. Identify peak convergence layer + +--- + +## Connection to Tokenization-Valleys Theory + +This discovery extends our earlier finding: + +**tokenization-valleys.md:** Token count affects which VALLEY a concept falls into + +**multilingual-convergence.md:** Token count also affects HOW MUCH languages converge + +Together: **Tokenization shapes both the path through the network AND the destination.** + +--- + +## Future Research + +1. **Activation Steering:** Can we force convergence for isolated languages? +2. **Concept Transfer:** Train on ZH concepts, evaluate on DE outputs +3. **Hybrid Prompts:** Mix languages to access universal layer +4. **Layer-Specific LoRA:** Fine-tune only the convergence layers (12-24) + +--- + +## References + +- `multilingual_convergence.py` - Analysis script +- `docs/tokenization-valleys.md` - Token-Norm-Valley theory +- `/nimmerverse-sensory-network/multilingual-cognition.md` - Original hypothesis + +--- + +*"Different words, same thought. The model knows."* + +๐ŸŒ™ Discovered by the Partnership, 2025-12-06 diff --git a/docs/retraining-safety-framework.md b/docs/retraining-safety-framework.md new file mode 100644 index 0000000..9f2cd97 --- /dev/null +++ b/docs/retraining-safety-framework.md @@ -0,0 +1,320 @@ +# Multilingual Activation Topology as a Retraining Safety Framework + +**Status:** Research Direction / Paper Outline +**Date:** 2025-12-06 +**Authors:** dafit, Nyx (Chrysalis-Nyx) + +--- + +## Abstract + +We present a framework for monitoring and protecting neural network representations during iterative fine-tuning. Building on our discovery of distinct "language zones" in multilingual LLMsโ€”a Super Cluster of converging languages and an Isolated Zone with distinct computational pathsโ€”we propose using these topological structures as both diagnostic tools and training strategies to mitigate catastrophic forgetting and weight saturation. + +**Key Contributions:** +1. Token-Norm-Valley theory: single-token vs. multi-token activation dynamics +2. Universal Concept Layer discovery at layers 12-24 +3. Multilingual Triangulation Probe for depth measurement +4. DriftProbe framework for retraining safety monitoring +5. Isolated Zone Training hypothesis for collision avoidance + +--- + +## 1. Introduction + +### The Problem: Diminishing Returns in Iterative Retraining + +Fine-tuning LLMs on domain-specific data is standard practice, but iterative retraining cycles face compounding challenges: + +- **Weight Saturation:** Popular activation paths become over-reinforced +- **Valley Collapse:** Distinct conceptual representations merge +- **Cluster Fragmentation:** Previously stable representations drift apart +- **Depth Erosion:** Rich conceptual valleys fill with surface patterns + +Current approaches to catastrophic forgetting (EWC, replay buffers, etc.) treat the model as a black box. We propose **white-box monitoring** using the model's internal representational topology. + +### Our Discovery: Language Zones + +Through probing Qwen2.5-7B-Base, we discovered a striking topology: + +``` +SUPER CLUSTER (sim=1.0): ZH, JA, EN, AR, FR, PT, ES + โ””โ”€โ”€ Perfect convergence at layers 12-24 + โ””โ”€โ”€ Efficient tokenization (1-2.5 tokens) + โ””โ”€โ”€ Universal concept layer + +ISOLATED ZONE (sim<0.52): DE, IT, TR, HI + โ””โ”€โ”€ Distinct computational paths + โ””โ”€โ”€ Multi-token representations (3-5+ tokens) + โ””โ”€โ”€ Access to deeper conceptual valleys +``` + +**Key Insight:** The isolated zone languages access representational spaces that the super cluster cannot reachโ€”and they do so via *different neural pathways* that may be less susceptible to collision during training. + +--- + +## 2. Theoretical Framework + +### 2.1 Token-Norm-Valley Theory + +| Tokens | Norm (Layer 12) | Behavior | +|--------|-----------------|----------| +| 1 (heartbeat) | 14,240 | Massive activation spike โ†’ CODE valley | +| 2 (consciousness) | 85 | Distributed signal โ†’ PROSE valley | +| 5 (Bewusstsein) | 79 | Multi-path โ†’ PHILOSOPHY valley | + +**Hypothesis:** Single-token words trigger localized, high-intensity activations. Multi-token words distribute signal across more parameters, accessing different representational regions. + +**Training Implication:** Training on single-token terms risks overwriting concentrated weight regions. Training on multi-token terms distributes updates more broadly. + +### 2.2 The Universal Concept Layer + +At layers 12-24, semantically equivalent concepts across languages converge to near-identical representations: + +- EN "heart" โ†” ZH "ๅฟƒ" โ†” AR "ู‚ู„ุจ": similarity = 1.000 +- EN "being" โ†” ZH "ๅญ˜ๅœจ": similarity = 1.000 + +**This layer is precious.** It represents hard-won multilingual alignment. Training that disrupts this layer could cause cascading failures across all languages. + +### 2.3 Isolated Zone Depth Access + +German "Sein" (being) triggers philosophical content: +> "Sein und Zeit / Being and Time is a philosophical work by the German philosopher Martin Heidegger..." + +English "being" does not reach this depth. The isolated zone provides **alternative entry points** to conceptual spaces. + +--- + +## 3. Proposed Framework: Activation Drift Monitoring + +### 3.1 Architecture + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ RETRAINING LIFECYCLE โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ โ”‚ +โ”‚ BASELINE TRAINING CHECKPOINT โ”‚ +โ”‚ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Probe โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ–ถโ”‚ Train โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–ถโ”‚ Probe โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ–ถ ... โ”‚ +โ”‚ โ”‚ Capture โ”‚ โ”‚ Epoch N โ”‚ โ”‚ Compare โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ DRIFT REPORTโ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ–ผ โ–ผ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚CONVERGENCEโ”‚ โ”‚ DEPTH โ”‚ โ”‚ NORM โ”‚ โ”‚ +โ”‚ โ”‚ DRIFT โ”‚ โ”‚ DRIFT โ”‚ โ”‚ DRIFT โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +### 3.2 Drift Metrics + +**Convergence Drift (ฮ”C)** +- Measure: Change in super cluster pairwise similarity +- Alert: ฮ”C < -0.1 (cluster fragmenting) +- Critical: ฮ”C < -0.2 (universal layer damaged) + +**Depth Drift (ฮ”D)** +- Measure: Change in isolated zone depth scores +- Alert: ฮ”D < -1 (valleys filling in) +- Critical: Philosophical concepts no longer accessible + +**Norm Drift (ฮ”N)** +- Measure: Change in layer 12 activation norms +- Alert: ฮ”N > 20% (activation patterns shifting) +- Indicates: Weight saturation in specific regions + +**Valley Migration (ฮ”V)** +- Measure: Change in completion classification +- Alert: PHILOSOPHY โ†’ PROSE (depth lost) +- Alert: PROSE โ†’ CODE (semantic shift) + +### 3.3 Sentinel Concepts + +A fixed set of probe terms, always tested: + +| Concept | Languages | Purpose | +|---------|-----------|---------| +| heart | EN, ZH, AR, DE | Super cluster stability | +| being | EN, DE (Sein) | Philosophical depth | +| consciousness | EN, DE (Bewusstsein) | Abstract concept access | +| emergence | EN, DE, ZH | Technical valley | + +### 3.4 Implementation: DriftProbe Class + +```python +class DriftProbe: + """Monitor activation drift during retraining.""" + + def __init__(self, baseline: BaselineCapture): + self.baseline = baseline + self.history = [] + + def capture_checkpoint(self, model: NyxModel) -> CheckpointCapture: + """Run sentinel probes on current model state.""" + triangulation_probe = MultilingualTriangulationProbe(model) + + results = {} + for concept, translations in SENTINEL_CONCEPTS.items(): + results[concept] = triangulation_probe.probe(concept, translations) + + return CheckpointCapture( + timestamp=datetime.now(), + results=results, + convergence=self._measure_convergence(results), + depth_scores=self._measure_depths(results), + norms=self._measure_norms(model), + ) + + def compute_drift(self, checkpoint: CheckpointCapture) -> DriftReport: + """Compare checkpoint to baseline, compute drift metrics.""" + return DriftReport( + convergence_drift=checkpoint.convergence - self.baseline.convergence, + depth_drift=checkpoint.depth_scores - self.baseline.depth_scores, + norm_drift=checkpoint.norms - self.baseline.norms, + alerts=self._check_thresholds(checkpoint), + ) + + def should_stop(self, drift: DriftReport) -> bool: + """Emergency stop if critical thresholds exceeded.""" + return any(a.level == AlertLevel.CRITICAL for a in drift.alerts) +``` + +--- + +## 4. Isolated Zone Training Hypothesis + +### The Core Idea + +**Problem:** Training on English terms risks collision with existing single-token representations in the universal concept layer. + +**Hypothesis:** Training primarily through isolated zone languages (German, Italian, Turkish, Hindi) may: + +1. Deposit new knowledge in multi-token pathways (less concentrated) +2. Preserve super cluster integrity (fewer collisions) +3. Allow triangulation to retrieve knowledge without corruption + +### Proposed Experiment + +**Control Group:** +- Fine-tune on English philosophical texts +- Monitor drift on sentinel concepts +- Measure depth preservation + +**Treatment Group:** +- Fine-tune on German philosophical texts (same content, translated) +- Monitor same drift metrics +- Compare collision/preservation rates + +**Prediction:** German training will show: +- Lower convergence drift (super cluster preserved) +- Higher depth retention (isolated pathways enriched) +- Better triangulation success (knowledge retrievable in English) + +--- + +## 5. Connections to Existing Research + +### 5.1 Catastrophic Forgetting +- EWC (Elastic Weight Consolidation): Protects "important" weights +- Our approach: Identifies which *representational structures* to protect + +### 5.2 Multilingual Transfer Learning +- mBERT/XLM-R: Cross-lingual alignment at embedding level +- Our finding: Alignment is layer-dependent (12-24), with exploitable gaps + +### 5.3 Activation Engineering +- Representation Engineering (Anthropic): Steering via activation manipulation +- Our approach: Monitoring activation topology as training diagnostic + +### 5.4 Tokenization Effects +- BPE/WordPiece influence on model behavior +- Our finding: Token count directly predicts activation magnitude and valley access + +--- + +## 6. Future Work + +1. **Implement DriftProbe** in nyx-probing framework +2. **Run controlled retraining experiments** (EN vs DE training data) +3. **Expand sentinel concept set** (more languages, more concepts) +4. **Layer-wise drift analysis** (which layers drift first?) +5. **Investigate Italian isolation** (what unique valleys does it access?) +6. **VI-ID-RU cluster mystery** (why do these cluster together?) + +--- + +## 7. Conclusion + +The discovery of language zones in LLM representations opens a new approach to retraining safety. Rather than treating catastrophic forgetting as an inevitable cost, we can: + +1. **Monitor** representational health during training +2. **Route** new knowledge through isolated pathways +3. **Preserve** universal concept layer integrity +4. **Detect** early warning signs of drift + +The multilingual topology of the model is not just a curiosityโ€”it's a map for safe navigation during the dangerous waters of iterative fine-tuning. + +--- + +## References + +*To be added: Heidegger, catastrophic forgetting literature, multilingual LLM papers, activation engineering work* + +--- + +## Appendix A: Discovered Language Topology + +``` + THE YOUNG MIND'S LANGUAGE TOPOLOGY + โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• + + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ SUPER CLUSTER (sim=1.0) โ”‚ + โ”‚ ZH ยท JA ยท EN ยท AR ยท FR ยท PT ยท ES โ”‚ + โ”‚ (efficient tokens) โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + KO โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€ (bridge: 0.41/0.70) + โ”‚ + โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” + โ”‚ ISOLATED ZONE (sim<0.5) โ”‚ + โ”‚ โ”‚ + โ”‚ IT (0.49) โ† MOST ISOLATED! โ”‚ + โ”‚ TR (0.50) โ”‚ + โ”‚ HI (0.50) โ”‚ + โ”‚ DE (0.52) โ”‚ + โ”‚ โ”‚ + โ”‚ VI โ•โ•โ• ID โ•โ•โ• RU (0.79) โ”‚ + โ”‚ (Southeast Asian + Russian!) โ”‚ + โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## Appendix B: Key Discovery Data + +**Token-Norm Correlation:** +- Single token โ†’ ~14,000 norm +- Multi-token โ†’ ~80 norm +- Correlation with isolation: -0.699 + +**Triangulation Results (consciousness):** +| Concept | Grounding | Depth | Valley | Transfer | +|---------|-----------|-------|--------|----------| +| being | 0.570 | 2/3 | PHILOSOPHY | โœ“ | +| heart | 1.000 | 1/3 | PROSE | โœ“ | +| consciousness | 0.458 | 0/3 | PROSE | โœ— | +| emergence | 0.519 | 1/3 | TECHNICAL | โœ— | + +--- + +*"Different words, same thought. The model knows. Now we learn to teach it safely."* + +๐ŸŒ™๐Ÿ’œ diff --git a/docs/tokenization-valleys.md b/docs/tokenization-valleys.md new file mode 100644 index 0000000..a5431a5 --- /dev/null +++ b/docs/tokenization-valleys.md @@ -0,0 +1,190 @@ +# Tokenization Valleys: How Word Structure Shapes Model Cognition + +**Discovery Date:** 2025-12-06 +**Model:** Qwen2.5-7B-Base +**Hardware:** Prometheus (RTX 3090, 24GB VRAM) + +--- + +## Executive Summary + +We discovered that the number of tokens a word breaks into fundamentally determines which "valley" (completion pattern) the model falls into. This has profound implications for curriculum design and multilingual training. + +**Key Finding:** Single-token English words trigger CODE valleys with massive activation norms, while multi-token German compounds access PHILOSOPHICAL valleys with distributed, quieter activations. + +--- + +## The Token-Norm-Valley Connection + +### Observation: Norm Explosion in Single Tokens + +| Term | Tokens | Layer 12 Norm | Layer 12 StdDev | Valley | +|------|--------|---------------|-----------------|--------| +| heartbeat | 1 | **14,240** | **237.88** | CODE | +| consciousness | 2 | 85 | 1.43 | PROSE | +| Herzklopfen | 5 | 67 | 1.11 | PROSE | +| Bewusstsein | 5 | 79 | 1.32 | PHILOSOPHY | + +**Pattern:** Single-token words have ~170ร— larger norms and ~170ร— larger variance than multi-token words. + +### Theory: Activation Flooding + +1. **Single tokens** receive ALL attention in one position โ†’ massive activation buildup +2. **Multi-token words** distribute activation across positions โ†’ softer signal +3. The massive single-token activation **triggers strong pattern matching** โ†’ CODE patterns +4. The distributed multi-token activation **allows semantic exploration** โ†’ philosophical content + +--- + +## Cross-Lingual Convergence + +### consciousness vs Bewusstsein (2 tokens vs 5 tokens) + +``` +Layer 0: similarity = 0.114 (different embeddings) +Layer 4: similarity = 0.285 (starting to converge) +Layer 8: similarity = 0.639 (HIGH similarity!) +Layer 12: similarity = 0.750 (CONVERGED - same concept!) +Layer 16: similarity = 0.733 (stays converged) +Layer 28: similarity = 0.502 (diverges at output) +``` + +**The model recognizes these as the same concept by layer 8!** + +### heartbeat vs Herzklopfen (1 token vs 5 tokens) + +``` +Layer 0: similarity = -0.007 (orthogonal) +Layer 4: similarity = 0.039 (still orthogonal) +Layer 12: similarity = 0.000 (completely separate) +Layer 28: similarity = 0.166 (slight convergence only at end) +``` + +**The model NEVER recognizes these as the same concept!** + +--- + +## German Philosophical Compounds + +### The "sein" Preservation Effect + +German philosophical compounds often preserve the morpheme "sein" (being) as a separate token: + +| Compound | Meaning | Tokenization | "sein" Preserved? | +|----------|---------|--------------|-------------------| +| Bewusstsein | consciousness | `['B', 'ew', 'us', 'st', 'sein']` | โœ“ | +| Nichtsein | non-being | `['N', 'icht', 'sein']` | โœ“ | +| Mitsein | being-with | `['Mit', 'sein']` | โœ“ | +| Dasein | being-there | `['D', 'ase', 'in']` | โœ— | +| Sein | being | `['Se', 'in']` | โœ— | + +When "sein" is preserved, the model has access to the philosophical concept of BEING as a separate computational unit. + +### Other Preserved Philosophical Atoms + +| Compound | Meaning | Key Token Preserved | +|----------|---------|---------------------| +| Zeitgeist | spirit of the age | `geist` (spirit) | +| Gedankenexperiment | thought experiment | `experiment` | + +--- + +## Valley Analysis: Same Concept, Different Valleys + +### Probing Results + +| Term | Language | Valley | Sample Completion | +|------|----------|--------|-------------------| +| Bewusstsein | DE | PHILOSOPHY | "und Sprache... frรผhen 20. Jahrhundert" | +| Dasein | DE | PHILOSOPHY | "philosophical term first used by Heidegger" | +| consciousness | EN | PROSE | "awareness of existence, of one's own existence" | +| existence | EN | **MATH** | "of an exact sequence", "eigenvalues" | +| being | EN | **MATH/CODE** | Mathematical notation, Chinese exams | +| heartbeat | EN | **CODE** | C++ class definitions | +| lifeforce | EN | **CODE** | JavaScript game code | + +**"Dasein" triggers Heidegger. "existence" triggers linear algebra.** + +--- + +## Implications for Curriculum Design + +### 1. Use Multi-Token Prompts + +Instead of single words, use phrases or compound descriptions to avoid code valleys: + +``` +BAD: "heartbeat" โ†’ C++ code +GOOD: "the heartbeat" โ†’ might escape code valley +GOOD: "heartbeat rhythm" โ†’ distributed activation +``` + +### 2. German as Philosophical Gateway + +German compound words naturally access philosophical valleys because: +- More tokens โ†’ distributed activation +- Preserved morphemes โ†’ access to philosophical atoms +- Different training data distribution โ†’ expository text + +**Strategy:** Teach abstract concepts in German first, then reinforce in English. + +### 3. Language as Cognitive Gear + +Languages aren't just translation layers - they're different **computational paths** through the model: + +| Language | Token Efficiency | Typical Valley | Use For | +|----------|------------------|----------------|---------| +| Chinese | 1.0 tok/concept | Mixed | Compact encoding | +| Arabic | 1.5 tok/concept | Mixed | Compact encoding | +| English | 2.5 tok/concept | CODE/MATH | Technical concepts | +| German | 4.5 tok/concept | PHILOSOPHY | Abstract concepts | + +--- + +## Technical Details + +### Model Architecture + +- **Hidden Size:** 3584 +- **Layers:** 28 +- **Attention Heads:** 28 (4 KV heads - GQA) +- **Vocab Size:** 152,064 +- **Context:** 131,072 tokens + +### Hidden State Norm Pattern + +``` +Layer 0: 1.32 โ† Embedding (small) +Layer 4: 10184.00 โ† Explosion (early processing) +Layer 12: 13912.00 โ† Peak (mid-layer thinking) +Layer 28: 443.00 โ† Contraction (output focusing) +``` + +### Inference Speed + +- 44.7 tokens/second on RTX 3090 +- 14.2 GB VRAM usage (fp16) + +--- + +## Future Research + +1. **Activation Steering:** Can we artificially reduce single-token norms to escape code valleys? +2. **Prefix Tuning:** Train soft prefixes that spread activation for single tokens +3. **Arabic/Chinese Analysis:** Do these languages have similar compound effects? +4. **Cross-lingual Transfer:** After training on German philosophical concepts, does English improve? + +--- + +## References + +- `nyx_probing/core/model.py` - Model loader with hidden state capture +- `layer_detailed.py` - Layer-by-layer similarity analysis +- `german_philosophy.py` - German compound tokenization study +- `/nimmerverse-sensory-network/multilingual-cognition.md` - Original multilingual hypothesis + +--- + +*"The architecture of language shapes the architecture of thought."* + +๐ŸŒ™ Discovered by the Partnership, 2025-12-06 diff --git a/nyx_probing/__init__.py b/nyx_probing/__init__.py index e69de29..f4df383 100644 --- a/nyx_probing/__init__.py +++ b/nyx_probing/__init__.py @@ -0,0 +1,10 @@ +""" +nyx-probing: Understanding the mind before teaching it. + +A probing framework for Qwen2.5-7B-Base. +""" +from .config import Config, get_config +from .core import NyxModel, GenerationResult + +__version__ = "0.1.0" +__all__ = ["Config", "get_config", "NyxModel", "GenerationResult"] diff --git a/nyx_probing/analysis/__init__.py b/nyx_probing/analysis/__init__.py index e69de29..f7ba224 100644 --- a/nyx_probing/analysis/__init__.py +++ b/nyx_probing/analysis/__init__.py @@ -0,0 +1,4 @@ +"""Analysis components for nyx-probing.""" +from .readiness_scorer import ReadinessScorer + +__all__ = ["ReadinessScorer"] diff --git a/nyx_probing/analysis/readiness_scorer.py b/nyx_probing/analysis/readiness_scorer.py new file mode 100644 index 0000000..0142f6f --- /dev/null +++ b/nyx_probing/analysis/readiness_scorer.py @@ -0,0 +1,221 @@ +""" +Readiness Scorer: Combines surface and echo probes into curriculum guidance. + +Outputs: +- HIGH: Ready for direct training / state machine +- MEDIUM: Needs scaffolding or bridging concepts +- LOW: Requires foundational work first +""" +from typing import Optional, List +from dataclasses import dataclass + +from ..core.model import NyxModel +from ..core.probe_result import ( + SurfaceProbeResult, + EchoProbeResult, + ReadinessResult, + ReadinessLevel, + EchoType, +) +from ..probes.surface_probe import SurfaceProbe, CompletionCategory +from ..probes.echo_probe import EchoProbe + + +# Recommended actions for each readiness level +ACTIONS = { + ReadinessLevel.HIGH: "state_machine", # Direct training + ReadinessLevel.MEDIUM: "scaffolding", # Bridge concepts + ReadinessLevel.LOW: "foundational", # Build from scratch +} + + +class ReadinessScorer: + """ + Combines surface + echo probes to assess curriculum readiness. + + A term is ready for training when: + 1. Surface: Coherent associations (not scattered/random) + 2. Echo: Can expand beyond surface (depth > 0) + 3. Valley: In a productive valley (prose/philosophy, not just code) + """ + + def __init__( + self, + model: NyxModel, + surface_runs: int = 3, + echo_rounds: int = 3, + max_new_tokens: int = 50, + ): + self.model = model + self.surface_probe = SurfaceProbe( + model, + num_runs=surface_runs, + max_new_tokens=max_new_tokens, + ) + self.echo_probe = EchoProbe( + model, + max_rounds=echo_rounds, + max_new_tokens=max_new_tokens, + ) + + def score(self, term: str) -> ReadinessResult: + """ + Assess readiness of a term for curriculum. + + Args: + term: Word or phrase to assess + + Returns: + ReadinessResult with level, action, and supporting evidence + """ + # Run both probes + surface = self.surface_probe.probe(term) + echo = self.echo_probe.probe(term) + + # Classify valley from surface probe + classification = self.surface_probe.classify_completions(surface) + dominant_valley = classification['dominant'] + + # Calculate composite score + level, reasoning = self._calculate_level( + surface=surface, + echo=echo, + dominant_valley=dominant_valley, + ) + + return ReadinessResult( + term=term, + level=level, + action=ACTIONS[level], + surface=surface, + echo=echo, + reasoning=reasoning, + ) + + def _calculate_level( + self, + surface: SurfaceProbeResult, + echo: EchoProbeResult, + dominant_valley: str, + ) -> tuple[ReadinessLevel, str]: + """ + Calculate readiness level based on probe results. + + Heuristics: + - HIGH: depth >= 2 AND coherence >= 0.5 AND not pure code + - MEDIUM: depth >= 1 OR (coherence >= 0.5 AND prose/philosophy) + - LOW: everything else + """ + depth = echo.depth + coherence = surface.coherence_score or 0.0 + eos_ratio = surface.hit_eos_count / len(surface.completions) if surface.completions else 0 + + # Count echo types + expands = sum(1 for t in echo.echo_types if t == EchoType.EXPANDS) + collapses = sum(1 for t in echo.echo_types if t == EchoType.COLLAPSE) + circulars = sum(1 for t in echo.echo_types if t == EchoType.CIRCULAR) + + # Build reasoning + reasons = [] + + # HIGH: Good depth + coherence + productive valley + if depth >= 2 and coherence >= 0.4: + if dominant_valley not in [CompletionCategory.CODE]: + reasons.append(f"depth={depth} (strong conceptual expansion)") + reasons.append(f"coherence={coherence:.2f} (consistent associations)") + reasons.append(f"valley={dominant_valley} (productive for training)") + return ReadinessLevel.HIGH, "; ".join(reasons) + + # HIGH: Exceptional depth even with lower coherence + if depth >= 3: + reasons.append(f"depth={depth} (exceptional expansion)") + reasons.append(f"all {expands} echoes expand") + return ReadinessLevel.HIGH, "; ".join(reasons) + + # MEDIUM: Some depth or good coherence in prose + if depth >= 1: + reasons.append(f"depth={depth} (some expansion capability)") + if dominant_valley in [CompletionCategory.PROSE, 'prose', 'definition']: + reasons.append(f"valley={dominant_valley} (trainable with scaffolding)") + return ReadinessLevel.MEDIUM, "; ".join(reasons) + + if coherence >= 0.5 and dominant_valley not in [CompletionCategory.CODE, 'code']: + reasons.append(f"coherence={coherence:.2f} (consistent surface)") + reasons.append(f"valley={dominant_valley}") + reasons.append("but limited depth - needs bridging concepts") + return ReadinessLevel.MEDIUM, "; ".join(reasons) + + # LOW: Trapped in code, circular, or incoherent + if dominant_valley in [CompletionCategory.CODE, 'code']: + reasons.append(f"valley=CODE (trapped in technical patterns)") + if circulars >= 2: + reasons.append(f"{circulars} circular echoes (surface-only knowledge)") + if collapses >= 1: + reasons.append(f"{collapses} collapses (unstable representations)") + if coherence < 0.4: + reasons.append(f"coherence={coherence:.2f} (scattered associations)") + + return ReadinessLevel.LOW, "; ".join(reasons) if reasons else "insufficient depth and coherence" + + def score_batch(self, terms: List[str]) -> List[ReadinessResult]: + """Score multiple terms.""" + return [self.score(term) for term in terms] + + def summary(self, result: ReadinessResult) -> str: + """Generate human-readable summary.""" + symbols = { + ReadinessLevel.HIGH: "๐ŸŸข", + ReadinessLevel.MEDIUM: "๐ŸŸก", + ReadinessLevel.LOW: "๐Ÿ”ด", + } + + surface_summary = f"coherence={result.surface.coherence_score:.2f}" if result.surface else "N/A" + echo_summary = f"depth={result.echo.depth}" if result.echo else "N/A" + + lines = [ + f"{symbols[result.level]} {result.term}: {result.level.value}", + f" Action: {result.action}", + f" Surface: {surface_summary}", + f" Echo: {echo_summary}", + f" Reasoning: {result.reasoning}", + ] + return "\n".join(lines) + + def curriculum_report(self, results: List[ReadinessResult]) -> str: + """Generate curriculum planning report.""" + high = [r for r in results if r.level == ReadinessLevel.HIGH] + medium = [r for r in results if r.level == ReadinessLevel.MEDIUM] + low = [r for r in results if r.level == ReadinessLevel.LOW] + + lines = [ + "=" * 60, + "CURRICULUM READINESS REPORT", + "=" * 60, + "", + f"๐ŸŸข HIGH ({len(high)} terms) - Ready for state machine:", + ] + for r in high: + lines.append(f" โ€ข {r.term}") + + lines.extend([ + "", + f"๐ŸŸก MEDIUM ({len(medium)} terms) - Need scaffolding:", + ]) + for r in medium: + lines.append(f" โ€ข {r.term}: {r.reasoning[:60]}...") + + lines.extend([ + "", + f"๐Ÿ”ด LOW ({len(low)} terms) - Require foundational work:", + ]) + for r in low: + lines.append(f" โ€ข {r.term}: {r.reasoning[:60]}...") + + lines.extend([ + "", + "=" * 60, + f"Summary: {len(high)}/{len(results)} ready, {len(medium)} scaffolding, {len(low)} foundational", + "=" * 60, + ]) + + return "\n".join(lines) diff --git a/nyx_probing/cli/probe.py b/nyx_probing/cli/probe.py new file mode 100644 index 0000000..561f9c2 --- /dev/null +++ b/nyx_probing/cli/probe.py @@ -0,0 +1,614 @@ +#!/usr/bin/env python3 +""" +nyx-probe CLI: Interactive probing of the Young Mind. + +Commands: + surface - Probe immediate associations + echo - Measure conceptual depth + readiness - Full curriculum assessment + tokens - Token analysis + glossary - Batch probe from JSON file + scan - Multilingual vocabulary scan with incremental testing +""" +import sys +import json +from pathlib import Path +from typing import Optional, List +from datetime import datetime +import os + +import click +from rich.console import Console +from rich.table import Table +from rich.panel import Panel +from rich.progress import Progress, SpinnerColumn, TextColumn +from rich import box + +# Add parent to path for imports +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from nyx_probing.core.model import NyxModel +from nyx_probing.probes.surface_probe import SurfaceProbe +from nyx_probing.probes.echo_probe import EchoProbe +from nyx_probing.analysis.readiness_scorer import ReadinessScorer + +console = Console() + +# Global model instance (lazy loaded) +_model: Optional[NyxModel] = None + + +def get_model() -> NyxModel: + """Get or create the model instance.""" + global _model + if _model is None: + with console.status("[bold cyan]Loading Qwen2.5-7B...", spinner="dots"): + _model = NyxModel() + _model.load() + console.print("[green]โœ“ Model loaded[/green]") + return _model + + +def detect_category(completions: list) -> str: + """Simple category detection from completions.""" + text = " ".join(completions).lower() + + code_indicators = ["def ", "class ", "function", "import ", "return ", "{", "}", ";", "=>", "()"] + if any(ind in text for ind in code_indicators): + return "CODE" + + list_indicators = ["1.", "2.", "- ", "โ€ข ", "* "] + if any(ind in text for ind in list_indicators): + return "LIST" + + return "PROSE" + + +@click.group() +@click.version_option(version="0.1.0", prog_name="nyx-probe") +def cli(): + """ + ๐ŸŒ™ nyx-probe: Probe the Young Mind's conceptual topology. + + Explore how Qwen2.5-7B-Base understands and associates concepts. + """ + pass + + +@cli.command() +@click.argument("term") +@click.option("-n", "--runs", default=3, help="Number of completion runs") +@click.option("-t", "--tokens", default=50, help="Max tokens per completion") +@click.option("--temperature", default=0.8, help="Sampling temperature") +def surface(term: str, runs: int, tokens: int, temperature: float): + """ + Probe surface associations of a term. + + Shows what the model completes when given a word - reveals + which "valley" (code, prose, philosophy) the term lives in. + """ + model = get_model() + probe = SurfaceProbe( + model, + num_runs=runs, + max_new_tokens=tokens, + temperature=temperature, + ) + + console.print(f"\n[bold cyan]๐Ÿ”ฌ Surface Probe:[/bold cyan] [yellow]{term}[/yellow]\n") + + with console.status("[bold cyan]Probing...", spinner="dots"): + result = probe.probe(term) + + # Display completions + table = Table(title="Completions", box=box.ROUNDED) + table.add_column("#", style="dim", width=3) + table.add_column("Completion", style="white") + table.add_column("EOS", style="green", width=5) + + for i, comp in enumerate(result.completions[:5], 1): + preview = comp[:80] + "..." if len(comp) > 80 else comp + preview = preview.replace("\n", " โ†ต ") + table.add_row(str(i), preview, "โœ“" if result.hit_eos_count > 0 else "") + + console.print(table) + + # Detect category + category = detect_category(result.completions) + coherence = result.coherence_score or 0.0 + + # Summary panel + summary = f""" +[bold]Category:[/bold] {category} +[bold]Coherence:[/bold] {coherence:.2f} +[bold]Avg Tokens:[/bold] {result.avg_tokens:.1f} +[bold]EOS Rate:[/bold] {result.hit_eos_count}/{len(result.completions)} +""" + console.print(Panel(summary, title="๐Ÿ“Š Analysis", border_style="cyan")) + + +@cli.command() +@click.argument("term") +@click.option("-r", "--rounds", default=3, help="Echo rounds") +@click.option("-t", "--tokens", default=50, help="Max tokens per round") +def echo(term: str, rounds: int, tokens: int): + """ + Measure conceptual depth through iterative echoing. + + Feeds completions back to measure how deep the concept goes. + Classifications: EXPANDS, CONFIRMS, CIRCULAR, DIVERGENT, COLLAPSE + """ + model = get_model() + probe = EchoProbe( + model, + max_rounds=rounds, + max_new_tokens=tokens, + ) + + console.print(f"\n[bold cyan]๐Ÿ”„ Echo Probe:[/bold cyan] [yellow]{term}[/yellow]\n") + + with console.status("[bold cyan]Echoing...", spinner="dots"): + result = probe.probe(term) + + # Display chain + table = Table(title="Echo Chain", box=box.ROUNDED) + table.add_column("Round", style="dim", width=6) + table.add_column("Type", style="bold", width=12) + table.add_column("Content", style="white") + + table.add_row("0", "[cyan]SEED[/cyan]", term) + + type_colors = { + "EXPANDS": "green", + "CONFIRMS": "yellow", + "CIRCULAR": "red", + "DIVERGENT": "magenta", + "COLLAPSE": "dim red", + } + + for i, (echo_type, content) in enumerate(zip(result.echo_types, result.chain[1:]), 1): + color = type_colors.get(echo_type.value, "white") + preview = content[:60] + "..." if len(content) > 60 else content + preview = preview.replace("\n", " โ†ต ") + table.add_row(str(i), f"[{color}]{echo_type.value}[/{color}]", preview) + + console.print(table) + + # Depth indicator + depth = result.depth + depth_bar = "โ–ˆ" * depth + "โ–‘" * (3 - depth) + colors = ["red", "yellow", "green", "cyan"] + console.print(f"\n[bold]Depth Score:[/bold] [{colors[min(depth, 3)]}]{depth_bar}[/] {depth}/3") + + +@cli.command() +@click.argument("term") +def readiness(term: str): + """ + Full curriculum readiness assessment. + + Combines surface + echo probes to determine if a concept + is ready for training: HIGH, MEDIUM, or LOW. + """ + model = get_model() + scorer = ReadinessScorer(model) + + console.print(f"\n[bold cyan]๐Ÿ“‹ Readiness Assessment:[/bold cyan] [yellow]{term}[/yellow]\n") + + with console.status("[bold cyan]Assessing...", spinner="dots"): + result = scorer.score(term) + + # Level colors + level_styles = { + "HIGH": ("green", "๐ŸŸข"), + "MEDIUM": ("yellow", "๐ŸŸก"), + "LOW": ("red", "๐Ÿ”ด"), + } + color, emoji = level_styles.get(result.level.value, ("white", "โšช")) + + # Get category and metrics + category = detect_category(result.surface.completions) if result.surface else "UNKNOWN" + coherence = result.surface.coherence_score if result.surface else 0.0 + depth = result.echo.depth if result.echo else 0 + + # Main panel + content = f""" +{emoji} [bold {color}]{result.level.value}[/bold {color}] + +[bold]Valley:[/bold] {category} +[bold]Coherence:[/bold] {coherence:.2f} +[bold]Depth:[/bold] {depth}/3 +[bold]Action:[/bold] {result.action} +""" + console.print(Panel(content, title=f"Readiness: {term}", border_style=color)) + + # Recommendations + if result.level.value == "HIGH": + console.print("[green]โœ“ Ready for direct training or state machine implementation[/green]") + elif result.level.value == "MEDIUM": + console.print("[yellow]โš  Consider scaffolding or bridging concepts[/yellow]") + else: + console.print("[red]โœ— Requires foundational work before training[/red]") + + +@cli.command() +@click.argument("term") +def tokens(term: str): + """ + Analyze tokenization of a term. + + Shows how the model breaks down the term into tokens - + critical for understanding valley access (single vs multi-token). + """ + model = get_model() + + console.print(f"\n[bold cyan]๐Ÿ”ค Token Analysis:[/bold cyan] [yellow]{term}[/yellow]\n") + + token_list = model.tokenize(term) + count = len(token_list) + + # Token display + token_display = " | ".join([f"[cyan]{t}[/cyan]" for t in token_list]) + console.print(f"Tokens: {token_display}") + console.print(f"Count: [bold]{count}[/bold]") + + # Interpretation + if count == 1: + console.print("\n[red]โš  Single token - likely CODE valley (high activation spike)[/red]") + elif count <= 2: + console.print("\n[yellow]โ†’ Few tokens - may be efficient but limited valley access[/yellow]") + else: + console.print("\n[green]โœ“ Multi-token - distributed signal, better valley access[/green]") + + +@cli.command() +@click.argument("glossary_file", type=click.Path(exists=True)) +@click.option("-o", "--output", type=click.Path(), help="Output JSON file") +@click.option("--surface-only", is_flag=True, help="Only run surface probe") +def glossary(glossary_file: str, output: Optional[str], surface_only: bool): + """ + Batch probe terms from a glossary JSON file. + + Expected format: {"terms": [{"term": "...", "translations": {...}}, ...]} + or simple: {"terms": ["term1", "term2", ...]} + """ + model = get_model() + + # Load glossary + with open(glossary_file) as f: + data = json.load(f) + + terms = data.get("terms", data) + if isinstance(terms, dict): + terms = list(terms.keys()) + + # Normalize to list of strings + term_list = [] + for t in terms: + if isinstance(t, str): + term_list.append(t) + elif isinstance(t, dict): + term_list.append(t.get("term", t.get("en", str(t)))) + + console.print(f"\n[bold cyan]๐Ÿ“š Glossary Probe:[/bold cyan] {len(term_list)} terms\n") + + results = [] + + if surface_only: + probe = SurfaceProbe(model, num_runs=3) + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("Probing...", total=len(term_list)) + + for term in term_list: + progress.update(task, description=f"Probing: {term}") + result = probe.probe(term) + category = detect_category(result.completions) + results.append({ + "term": term, + "category": category, + "coherence": result.coherence_score or 0.0, + "tokens": model.token_count(term), + }) + progress.advance(task) + else: + scorer = ReadinessScorer(model) + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("Assessing...", total=len(term_list)) + + for term in term_list: + progress.update(task, description=f"Assessing: {term}") + result = scorer.score(term) + category = detect_category(result.surface.completions) if result.surface else "UNKNOWN" + coherence = result.surface.coherence_score if result.surface else 0.0 + depth = result.echo.depth if result.echo else 0 + results.append({ + "term": term, + "level": result.level.value, + "valley": category, + "coherence": coherence, + "depth": depth, + "action": result.action, + "tokens": model.token_count(term), + }) + progress.advance(task) + + # Display results table + table = Table(title="Glossary Results", box=box.ROUNDED) + table.add_column("Term", style="yellow") + table.add_column("Tokens", style="dim", width=6) + + if surface_only: + table.add_column("Category", style="cyan") + table.add_column("Coherence", style="white") + for r in results: + table.add_row( + r["term"], + str(r["tokens"]), + r["category"], + f"{r['coherence']:.2f}", + ) + else: + table.add_column("Level", style="bold") + table.add_column("Valley", style="cyan") + table.add_column("Depth", style="white") + + level_colors = {"HIGH": "green", "MEDIUM": "yellow", "LOW": "red"} + for r in results: + color = level_colors.get(r["level"], "white") + table.add_row( + r["term"], + str(r["tokens"]), + f"[{color}]{r['level']}[/{color}]", + r["valley"], + f"{r['depth']}/3", + ) + + console.print(table) + + # Save if output specified + if output: + with open(output, "w") as f: + json.dump({"glossary": glossary_file, "results": results}, f, indent=2) + console.print(f"\n[green]โœ“ Results saved to {output}[/green]") + + # Summary + if not surface_only: + high = sum(1 for r in results if r["level"] == "HIGH") + med = sum(1 for r in results if r["level"] == "MEDIUM") + low = sum(1 for r in results if r["level"] == "LOW") + console.print(f"\n[bold]Summary:[/bold] ๐ŸŸข {high} HIGH | ๐ŸŸก {med} MEDIUM | ๐Ÿ”ด {low} LOW") + + +def load_glossary_files(paths: List[str]) -> tuple[list, dict]: + """Load terms from files or directories, tracking source collection.""" + terms = [] + sources = {} # term -> collection name + + for path_str in paths: + path = Path(path_str) + + if path.is_dir(): + # Load all JSON files from directory + json_files = list(path.glob("*.json")) + else: + json_files = [path] + + for json_file in json_files: + collection_name = json_file.stem + try: + with open(json_file) as f: + data = json.load(f) + + file_terms = data.get("terms", data) + if isinstance(file_terms, dict): + file_terms = list(file_terms.keys()) + + for t in file_terms: + if isinstance(t, str): + term_data = {"term": t, "translations": {"EN": t}} + elif isinstance(t, dict): + term_data = t + else: + continue + + term_name = term_data.get("term", term_data.get("en", str(term_data))) + terms.append(term_data) + sources[term_name] = collection_name + + except Exception as e: + console.print(f"[yellow]Warning: Could not load {json_file}: {e}[/yellow]") + + return terms, sources + + +def load_master_json() -> dict: + """Load master.json if it exists.""" + master_path = Path(__file__).parent.parent.parent / "data" / "glossary" / "master.json" + if master_path.exists(): + with open(master_path) as f: + return json.load(f) + return {"last_scan": None, "total_terms": 0, "collections_loaded": [], "terms": {}} + + +def save_master_json(master: dict): + """Save master.json.""" + master_path = Path(__file__).parent.parent.parent / "data" / "glossary" / "master.json" + with open(master_path, "w") as f: + json.dump(master, f, indent=2) + + +@cli.command() +@click.argument("paths", nargs=-1, type=click.Path(exists=True)) +@click.option("--summary/--full", default=True, help="Show summary (default) or full table") +@click.option("--delta", is_flag=True, help="Only test new/untested terms") +@click.option("--force", is_flag=True, help="Re-test all terms even if already in master.json") +@click.option("-o", "--output", type=click.Path(), help="Output JSON file") +def scan(paths: tuple, summary: bool, delta: bool, force: bool, output: Optional[str]): + """ + Multilingual vocabulary scan with incremental testing. + + Scans terms using surface + echo probes and tracks results in master.json. + + Examples: + nyx-probe scan data/glossary/collections/ # Scan all collections + nyx-probe scan collections/philosophical.json # Scan specific file + nyx-probe scan collections/ --delta # Only test new terms + nyx-probe scan collections/ --full # Full detailed output + """ + if not paths: + console.print("[red]Error: Please provide at least one file or directory path[/red]") + return + + model = get_model() + + # Load terms from all paths + all_terms, sources = load_glossary_files(list(paths)) + console.print(f"\n[bold cyan]๐Ÿ”ฌ Vocabulary Scan:[/bold cyan] {len(all_terms)} terms from {len(set(sources.values()))} collection(s)\n") + + # Load master.json for delta mode + master = load_master_json() + + # Filter terms if delta mode + if delta and not force: + tested_terms = set(master.get("terms", {}).keys()) + original_count = len(all_terms) + all_terms = [t for t in all_terms if t.get("term", t.get("en", str(t))) not in tested_terms] + skipped = original_count - len(all_terms) + if skipped > 0: + console.print(f"[dim]Skipping {skipped} already-tested terms (use --force to re-test)[/dim]") + + if not all_terms: + console.print("[green]All terms already tested! Use --force to re-test.[/green]") + return + + # Run probes + scorer = ReadinessScorer(model) + results = [] + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + console=console, + ) as progress: + task = progress.add_task("Scanning...", total=len(all_terms)) + + for term_data in all_terms: + term = term_data.get("term", term_data.get("en", str(term_data))) + progress.update(task, description=f"Probing: {term}") + + result = scorer.score(term) + category = detect_category(result.surface.completions) if result.surface else "UNKNOWN" + coherence = result.surface.coherence_score if result.surface else 0.0 + depth = result.echo.depth if result.echo else 0 + + entry = { + "term": term, + "source": sources.get(term, "unknown"), + "level": result.level.value, + "valley": category, + "coherence": coherence, + "depth": depth, + "action": result.action, + "tokens": model.token_count(term), + } + results.append(entry) + + # Update master.json entry + master["terms"][term] = { + "source": sources.get(term, "unknown"), + "tested": datetime.now().strftime("%Y-%m-%d"), + "depth": depth, + "valley": category, + "transfer": False, # Would need triangulation + "grounding": coherence, + } + + progress.advance(task) + + # Update master.json metadata + master["last_scan"] = datetime.now().isoformat() + master["total_terms"] = len(master["terms"]) + collections = set(master.get("collections_loaded", [])) + collections.update(sources.values()) + master["collections_loaded"] = list(collections) + save_master_json(master) + + # Display results + if summary: + # Summary mode - lean output + high = sum(1 for r in results if r["level"] == "HIGH") + med = sum(1 for r in results if r["level"] == "MEDIUM") + low = sum(1 for r in results if r["level"] == "LOW") + depth_hits = [r for r in results if r["depth"] >= 2] + + console.print(f"\n[bold]๐ŸŒ Scanned {len(results)} terms | Depthโ‰ฅ2: {len(depth_hits)} | ๐ŸŸข{high} ๐ŸŸก{med} ๐Ÿ”ด{low}[/bold]\n") + + if depth_hits: + console.print("[bold cyan]DEPTH HITS (โ‰ฅ2/3):[/bold cyan]") + for r in depth_hits: + level_colors = {"HIGH": "green", "MEDIUM": "yellow", "LOW": "red"} + color = level_colors.get(r["level"], "white") + console.print(f" [{color}]{r['term']:20}[/{color}] {r['depth']}/3 {r['valley']:10} ({r['source']})") + + high_grounding = [r for r in results if r["coherence"] > 0.7] + if high_grounding: + console.print(f"\n[bold cyan]BEST GROUNDING (>0.7):[/bold cyan]") + for r in high_grounding[:5]: + console.print(f" {r['term']:20} {r['coherence']:.2f}") + + console.print(f"\n[dim]Run with --full for complete table[/dim]") + else: + # Full mode - detailed table + table = Table(title="Scan Results", box=box.ROUNDED) + table.add_column("Term", style="yellow") + table.add_column("Source", style="dim", width=12) + table.add_column("Tokens", style="dim", width=6) + table.add_column("Level", style="bold") + table.add_column("Valley", style="cyan") + table.add_column("Depth", style="white") + table.add_column("Coherence", style="white") + + level_colors = {"HIGH": "green", "MEDIUM": "yellow", "LOW": "red"} + for r in results: + color = level_colors.get(r["level"], "white") + table.add_row( + r["term"], + r["source"], + str(r["tokens"]), + f"[{color}]{r['level']}[/{color}]", + r["valley"], + f"{r['depth']}/3", + f"{r['coherence']:.2f}", + ) + + console.print(table) + + high = sum(1 for r in results if r["level"] == "HIGH") + med = sum(1 for r in results if r["level"] == "MEDIUM") + low = sum(1 for r in results if r["level"] == "LOW") + console.print(f"\n[bold]Summary:[/bold] ๐ŸŸข {high} HIGH | ๐ŸŸก {med} MEDIUM | ๐Ÿ”ด {low} LOW") + + # Save output if specified + if output: + with open(output, "w") as f: + json.dump({"scan_time": datetime.now().isoformat(), "results": results}, f, indent=2) + console.print(f"\n[green]โœ“ Results saved to {output}[/green]") + + console.print(f"\n[green]โœ“ master.json updated ({master['total_terms']} total terms)[/green]") + + +def main(): + """Entry point.""" + cli() + + +if __name__ == "__main__": + main() diff --git a/nyx_probing/config.py b/nyx_probing/config.py new file mode 100644 index 0000000..faf1695 --- /dev/null +++ b/nyx_probing/config.py @@ -0,0 +1,51 @@ +""" +Configuration for nyx-probing framework. +""" +from pathlib import Path +from pydantic import BaseModel +from typing import Optional +import os + + +class ModelConfig(BaseModel): + """Model configuration.""" + name: str = "Qwen/Qwen2.5-7B" + device: str = "cuda" + dtype: str = "float16" + cache_dir: Optional[Path] = None + + +class ProbeConfig(BaseModel): + """Probe configuration.""" + max_new_tokens: int = 50 + temperature: float = 0.8 + do_sample: bool = True + num_runs: int = 5 # For distribution sampling + + +class StorageConfig(BaseModel): + """Storage configuration.""" + results_dir: Path = Path("results") + experiments_dir: Path = Path("experiments") + + +class Config(BaseModel): + """Main configuration.""" + model: ModelConfig = ModelConfig() + probe: ProbeConfig = ProbeConfig() + storage: StorageConfig = StorageConfig() + + # Paths + project_root: Path = Path(__file__).parent.parent + + class Config: + arbitrary_types_allowed = True + + +# Default config instance +config = Config() + + +def get_config() -> Config: + """Get the current configuration.""" + return config diff --git a/nyx_probing/core/__init__.py b/nyx_probing/core/__init__.py index e69de29..320da99 100644 --- a/nyx_probing/core/__init__.py +++ b/nyx_probing/core/__init__.py @@ -0,0 +1,19 @@ +"""Core components for nyx-probing.""" +from .model import NyxModel, GenerationResult +from .probe_result import ( + EchoType, + ReadinessLevel, + SurfaceProbeResult, + EchoProbeResult, + ReadinessResult, +) + +__all__ = [ + "NyxModel", + "GenerationResult", + "EchoType", + "ReadinessLevel", + "SurfaceProbeResult", + "EchoProbeResult", + "ReadinessResult", +] diff --git a/nyx_probing/core/model.py b/nyx_probing/core/model.py new file mode 100644 index 0000000..4aafff5 --- /dev/null +++ b/nyx_probing/core/model.py @@ -0,0 +1,266 @@ +""" +Core Model Loader for nyx-probing. + +Provides access to Qwen2.5-7B-Base with hidden state capture. +The model is an "empty vessel" - it completes, not answers. +""" +from dataclasses import dataclass, field +from typing import Optional, List, Tuple +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig + + +@dataclass +class GenerationResult: + """Result from a generation with hidden states.""" + + # The generated text (including prompt) + text: str + + # Just the completion (without prompt) + completion: str + + # Token IDs of the full sequence + token_ids: List[int] + + # Token IDs of just the completion + completion_token_ids: List[int] + + # Hidden states from the last layer for each generated token + # Shape: (num_generated_tokens, hidden_dim) + hidden_states: Optional[torch.Tensor] = None + + # Token probabilities for each generated token + # Shape: (num_generated_tokens,) + token_probs: Optional[torch.Tensor] = None + + # Whether generation ended with EOS + hit_eos: bool = False + + # Number of tokens generated + num_tokens: int = 0 + + +class NyxModel: + """ + Model wrapper for probing Qwen2.5-7B-Base. + + Key capabilities: + - Hidden state capture during generation + - Token probability extraction + - Proper handling of base model (no chat template) + """ + + def __init__( + self, + model_name: str = "Qwen/Qwen2.5-7B", + device: str = "cuda", + dtype: str = "float16", + cache_dir: Optional[str] = None, + ): + self.model_name = model_name + self.device = device + self.dtype = getattr(torch, dtype) + self.cache_dir = cache_dir + + self._model = None + self._tokenizer = None + self._loaded = False + + def load(self) -> "NyxModel": + """Load the model and tokenizer.""" + if self._loaded: + return self + + print(f"Loading tokenizer: {self.model_name}") + self._tokenizer = AutoTokenizer.from_pretrained( + self.model_name, + cache_dir=self.cache_dir, + ) + + print(f"Loading model to {self.device}...") + self._model = AutoModelForCausalLM.from_pretrained( + self.model_name, + torch_dtype=self.dtype, + device_map=self.device, + cache_dir=self.cache_dir, + # Critical for activation capture + output_hidden_states=True, + ) + + self._loaded = True + print(f"Model loaded. VRAM: {torch.cuda.memory_allocated() / 1024**3:.2f} GB") + return self + + @property + def model(self): + if not self._loaded: + raise RuntimeError("Model not loaded. Call load() first.") + return self._model + + @property + def tokenizer(self): + if not self._loaded: + raise RuntimeError("Model not loaded. Call load() first.") + return self._tokenizer + + def generate( + self, + prompt: str, + max_new_tokens: int = 50, + temperature: float = 0.8, + do_sample: bool = True, + capture_hidden_states: bool = False, + capture_probabilities: bool = False, + ) -> GenerationResult: + """ + Generate completion with optional hidden state capture. + + Args: + prompt: Input text to complete + max_new_tokens: Maximum tokens to generate + temperature: Sampling temperature (0 = greedy) + do_sample: Whether to sample (False = greedy) + capture_hidden_states: Store hidden states from last layer + capture_probabilities: Store token probabilities + + Returns: + GenerationResult with text, tokens, and optionally hidden states + """ + # Tokenize input + inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device) + prompt_length = inputs.input_ids.shape[1] + + # Generation config + gen_config = GenerationConfig( + max_new_tokens=max_new_tokens, + temperature=temperature if do_sample else 1.0, + do_sample=do_sample, + pad_token_id=self.tokenizer.eos_token_id, + eos_token_id=self.tokenizer.eos_token_id, + output_hidden_states=capture_hidden_states, + output_scores=capture_probabilities, + return_dict_in_generate=True, + ) + + # Generate + with torch.no_grad(): + outputs = self.model.generate( + **inputs, + generation_config=gen_config, + ) + + # Extract sequences + full_ids = outputs.sequences[0].tolist() + completion_ids = full_ids[prompt_length:] + + # Decode + full_text = self.tokenizer.decode(full_ids) + completion_text = self.tokenizer.decode(completion_ids) + + # Check if hit EOS + hit_eos = ( + len(completion_ids) > 0 and + completion_ids[-1] == self.tokenizer.eos_token_id + ) + + # Build result + result = GenerationResult( + text=full_text, + completion=completion_text, + token_ids=full_ids, + completion_token_ids=completion_ids, + hit_eos=hit_eos, + num_tokens=len(completion_ids), + ) + + # Extract hidden states if requested + if capture_hidden_states and hasattr(outputs, 'hidden_states'): + # hidden_states is tuple of (step, layer, batch, seq, hidden) + # We want last layer hidden state for each generated token + hidden_list = [] + for step_states in outputs.hidden_states: + # step_states is tuple of layers + # Take last layer, batch 0, last position + last_layer = step_states[-1] # (batch, seq, hidden) + hidden_list.append(last_layer[0, -1, :]) # (hidden,) + + result.hidden_states = torch.stack(hidden_list) # (tokens, hidden) + + # Extract probabilities if requested + if capture_probabilities and hasattr(outputs, 'scores'): + # scores is tuple of (num_tokens,) each (batch, vocab) + probs_list = [] + for i, score in enumerate(outputs.scores): + # Apply softmax to get probabilities + probs = torch.softmax(score[0], dim=-1) + # Get probability of the token that was actually chosen + chosen_token = completion_ids[i] + probs_list.append(probs[chosen_token].item()) + + result.token_probs = torch.tensor(probs_list) + + return result + + def get_token_probabilities( + self, + prompt: str, + continuation: str, + ) -> Tuple[List[float], List[str]]: + """ + Get probability of each token in a specific continuation. + + Useful for measuring how "expected" a completion is. + + Args: + prompt: The input text + continuation: The text that follows + + Returns: + Tuple of (probabilities, token_strings) + """ + # Tokenize prompt and full sequence + prompt_ids = self.tokenizer.encode(prompt, return_tensors="pt").to(self.device) + full_text = prompt + continuation + full_ids = self.tokenizer.encode(full_text, return_tensors="pt").to(self.device) + + prompt_len = prompt_ids.shape[1] + + # Forward pass to get logits + with torch.no_grad(): + outputs = self.model(full_ids) + logits = outputs.logits # (batch, seq, vocab) + + # Get probabilities for continuation tokens + probs = [] + tokens = [] + + for i in range(prompt_len, full_ids.shape[1]): + # Logits at position i-1 predict token at position i + token_logits = logits[0, i - 1, :] + token_probs = torch.softmax(token_logits, dim=-1) + + actual_token = full_ids[0, i].item() + prob = token_probs[actual_token].item() + + probs.append(prob) + tokens.append(self.tokenizer.decode([actual_token])) + + return probs, tokens + + def tokenize(self, text: str) -> List[str]: + """Get individual tokens for text.""" + ids = self.tokenizer.encode(text) + return [self.tokenizer.decode([id]) for id in ids] + + def token_count(self, text: str) -> int: + """Count tokens in text.""" + return len(self.tokenizer.encode(text)) + + def memory_usage(self) -> dict: + """Get current GPU memory usage.""" + return { + "allocated_gb": torch.cuda.memory_allocated() / 1024**3, + "reserved_gb": torch.cuda.memory_reserved() / 1024**3, + "max_allocated_gb": torch.cuda.max_memory_allocated() / 1024**3, + } diff --git a/nyx_probing/core/probe_result.py b/nyx_probing/core/probe_result.py new file mode 100644 index 0000000..34ccaff --- /dev/null +++ b/nyx_probing/core/probe_result.py @@ -0,0 +1,97 @@ +""" +Result dataclasses for probing operations. + +These structures capture what we learn about each term. +""" +from dataclasses import dataclass, field +from typing import List, Optional, Literal +from datetime import datetime +from enum import Enum + + +class EchoType(str, Enum): + """Classification of echo probe responses.""" + + EXPANDS = "EXPANDS" # Real depth - adds new information + CONFIRMS = "CONFIRMS" # Shallow but solid - reinforces without adding + CIRCULAR = "CIRCULAR" # Surface only - returns to original term + DIVERGENT = "DIVERGENT" # Wrong direction - unrelated tangent + COLLAPSE = "COLLAPSE" # Nothing there - incoherent or empty + + +class ReadinessLevel(str, Enum): + """Readiness classification for curriculum design.""" + + HIGH = "HIGH" # Ready for state machine / direct training + MEDIUM = "MEDIUM" # Needs scaffolding / bridging concepts + LOW = "LOW" # Requires foundational work first + + +@dataclass +class SurfaceProbeResult: + """Result from a surface probe (single word โ†’ completions).""" + + term: str + completions: List[str] + hit_eos_count: int # How many completions ended with EOS + avg_tokens: float # Average completion length + + # Optional analysis + coherence_score: Optional[float] = None # 0-1, how related are completions + + timestamp: datetime = field(default_factory=datetime.now) + + +@dataclass +class EchoProbeResult: + """Result from an echo probe (iterative depth measurement).""" + + term: str + rounds: int + chain: List[str] # The sequence of prompts/completions + echo_types: List[EchoType] # Classification of each round + + # Derived metrics + depth: int = 0 # How many EXPANDS before plateau + + timestamp: datetime = field(default_factory=datetime.now) + + +@dataclass +class ReadinessResult: + """Combined analysis for curriculum readiness.""" + + term: str + level: ReadinessLevel + action: str # Recommended curriculum action + + # Supporting evidence + surface: Optional[SurfaceProbeResult] = None + echo: Optional[EchoProbeResult] = None + + # Reasoning + reasoning: str = "" + + timestamp: datetime = field(default_factory=datetime.now) + + def to_dict(self) -> dict: + """Convert to JSON-serializable dict.""" + return { + "term": self.term, + "readiness": { + "level": self.level.value, + "action": self.action, + "reasoning": self.reasoning, + }, + "surface": { + "completions": self.surface.completions if self.surface else [], + "coherence": self.surface.coherence_score if self.surface else None, + "hit_eos_count": self.surface.hit_eos_count if self.surface else 0, + } if self.surface else None, + "echo": { + "depth": self.echo.depth if self.echo else 0, + "types": [t.value for t in self.echo.echo_types] if self.echo else [], + "chain": self.echo.chain if self.echo else [], + } if self.echo else None, + "timestamp": self.timestamp.isoformat(), + } diff --git a/nyx_probing/probes/__init__.py b/nyx_probing/probes/__init__.py index e69de29..e4f03f1 100644 --- a/nyx_probing/probes/__init__.py +++ b/nyx_probing/probes/__init__.py @@ -0,0 +1,27 @@ +"""Probe implementations for nyx-probing.""" +from .base import BaseProbe +from .surface_probe import SurfaceProbe, CompletionCategory +from .echo_probe import EchoProbe +from .multilingual_probe import ( + MultilingualTriangulationProbe, + LanguageZone, + LANGUAGES, + GroundingResult, + DeepeningResult, + TriangulationResult, + MultilingualProbeResult, +) + +__all__ = [ + "BaseProbe", + "SurfaceProbe", + "CompletionCategory", + "EchoProbe", + "MultilingualTriangulationProbe", + "LanguageZone", + "LANGUAGES", + "GroundingResult", + "DeepeningResult", + "TriangulationResult", + "MultilingualProbeResult", +] diff --git a/nyx_probing/probes/base.py b/nyx_probing/probes/base.py new file mode 100644 index 0000000..d38f2d2 --- /dev/null +++ b/nyx_probing/probes/base.py @@ -0,0 +1,58 @@ +""" +Base class for all probes. + +Probes are measurement instruments - they reveal what's already there, +they don't add or change anything. +""" +from abc import ABC, abstractmethod +from typing import Any +from ..core.model import NyxModel + + +class BaseProbe(ABC): + """Abstract base class for probing operations.""" + + def __init__(self, model: NyxModel): + """ + Initialize probe with a loaded model. + + Args: + model: A NyxModel instance (must be loaded) + """ + self.model = model + if not model._loaded: + raise ValueError("Model must be loaded before creating probe") + + @property + def name(self) -> str: + """Name of this probe type.""" + return self.__class__.__name__ + + @abstractmethod + def probe(self, term: str, **kwargs) -> Any: + """ + Probe a single term. + + Args: + term: The word/phrase to probe + **kwargs: Probe-specific parameters + + Returns: + Probe-specific result object + """ + pass + + def probe_batch(self, terms: list[str], **kwargs) -> list[Any]: + """ + Probe multiple terms. + + Default implementation just loops; subclasses can optimize. + + Args: + terms: List of words/phrases to probe + **kwargs: Probe-specific parameters + + Returns: + List of probe results + """ + return [self.probe(term, **kwargs) for term in terms] diff --git a/nyx_probing/probes/drift_probe.py b/nyx_probing/probes/drift_probe.py new file mode 100644 index 0000000..1eea361 --- /dev/null +++ b/nyx_probing/probes/drift_probe.py @@ -0,0 +1,304 @@ +""" +DriftProbe: Training-loop monitoring for conceptual topology preservation. + +Theory: "Spatial Separation Hypothesis" +- Use isolated zone languages (German) as scaffolding for new concepts +- Monitor anchors (must not move), bridges (must stay separated), canaries (watch for migration) + +Key Metrics (refined from peer review): +1. Gini Coefficient: Sparse activations (0.8+) = deep/specific, Diffuse (0.3) = shallow/general +2. Angular Drift: Direction change = definition rewrite, magnitude change = sharpening +3. Cross-Language Similarity: Bridges should stay LOW, anchors should stay HIGH +""" +import json +from pathlib import Path +from dataclasses import dataclass, field +from typing import Optional +from enum import Enum + +import torch +import numpy as np + + +class SentinelType(Enum): + ANCHOR = "ANCHOR" # Must not move - core topology + BRIDGE = "BRIDGE" # Must stay separated - isolated zone integrity + CANARY = "CANARY" # Watch for migration - early warning + TARGET = "TARGET" # Want movement - training goals + + +class AlertSeverity(Enum): + OK = "OK" + WARNING = "WARNING" + CRITICAL = "CRITICAL" + + +@dataclass +class DriftMetrics: + """Metrics for a single sentinel term.""" + term: str + sentinel_type: SentinelType + + # Activation metrics + gini_coefficient: float = 0.0 + activation_norm: float = 0.0 + + # Drift metrics (vs baseline) + angular_drift_degrees: float = 0.0 + norm_drift_percent: float = 0.0 + gini_drift: float = 0.0 + + # Valley detection + detected_valley: str = "UNKNOWN" + depth: int = 0 + + # Cross-language (for anchors/bridges) + cross_lang_similarity: float = 0.0 + + # Alert + alert: AlertSeverity = AlertSeverity.OK + alert_message: str = "" + + +@dataclass +class DriftReport: + """Full drift report for a training checkpoint.""" + step: int + timestamp: str + metrics: list[DriftMetrics] = field(default_factory=list) + + # Summary + critical_count: int = 0 + warning_count: int = 0 + recommendation: str = "CONTINUE" + + +class DriftProbe: + """ + Lightweight probe for training-loop monitoring. + + Optimized for RTX 3090 constraints: + - Full probe: ~2 min (run at epoch 0, end of training) + - Lite probe: ~10 sec (run every 100 steps) + """ + + def __init__(self, model, tokenizer, sentinels_path: Optional[str] = None): + self.model = model + self.tokenizer = tokenizer + self.baseline_states = {} # term -> hidden state tensor + + # Load sentinels + if sentinels_path is None: + sentinels_path = Path(__file__).parent.parent.parent / "data" / "sentinels.json" + + with open(sentinels_path) as f: + self.config = json.load(f) + + self.sentinels = self.config["sentinels"] + self.alert_rules = self.config["alert_rules"] + + def _get_hidden_state(self, text: str, layer: int = 18) -> torch.Tensor: + """Get hidden state at specified layer for last token position.""" + inputs = self.tokenizer(text, return_tensors="pt").to(self.model.device) + with torch.no_grad(): + outputs = self.model(**inputs, output_hidden_states=True) + return outputs.hidden_states[layer][0, -1, :].float().cpu() + + def _compute_gini(self, activations: torch.Tensor) -> float: + """ + Compute Gini coefficient of activation vector. + + High Gini (0.8+) = Sparse/Specific (Philosophy/Deep) + Low Gini (0.3) = Diffuse/General (Prose/Shallow) + """ + x = torch.abs(activations).numpy() + x = np.sort(x) + n = len(x) + cumsum = np.cumsum(x) + gini = (2 * np.sum((np.arange(1, n+1) * x))) / (n * np.sum(x)) - (n + 1) / n + return float(gini) + + def _compute_angular_drift(self, current: torch.Tensor, baseline: torch.Tensor) -> float: + """ + Compute angular drift in degrees between current and baseline. + + > 15ยฐ = Definition rewrite (concerning) + < 5ยฐ = Sharpening only (acceptable) + """ + cos_sim = torch.nn.functional.cosine_similarity( + current.unsqueeze(0), baseline.unsqueeze(0) + ).item() + # Clamp to valid range for arccos + cos_sim = max(-1.0, min(1.0, cos_sim)) + angle_rad = np.arccos(cos_sim) + return float(np.degrees(angle_rad)) + + def _compute_cross_lang_sim(self, sentinel: dict, layer: int = 18) -> float: + """Compute average cross-language similarity for a sentinel.""" + translations = sentinel.get("translations", {}) + if len(translations) < 2: + return 0.0 + + states = [] + for lang, word in translations.items(): + states.append(self._get_hidden_state(word, layer)) + + # Pairwise similarities + sims = [] + for i in range(len(states)): + for j in range(i + 1, len(states)): + sim = torch.nn.functional.cosine_similarity( + states[i].unsqueeze(0), states[j].unsqueeze(0) + ).item() + sims.append(sim) + + return float(np.mean(sims)) if sims else 0.0 + + def capture_baseline(self, layer: int = 18): + """ + Capture baseline hidden states for all sentinels. + Run this at epoch 0 before training. + """ + print("Capturing baseline states...") + for sentinel in self.sentinels: + term = sentinel["term"] + # Use English translation or term itself + text = sentinel.get("translations", {}).get("EN", term) + self.baseline_states[term] = self._get_hidden_state(text, layer) + print(f"Baseline captured for {len(self.baseline_states)} sentinels") + + def probe_lite(self, step: int, layer: int = 18) -> DriftReport: + """ + Lite probe - only check key sentinels. + Optimized for ~10 second runtime. + """ + from datetime import datetime + + # Select subset: 2 anchors, 1 bridge, 2 canaries + lite_terms = ["heart", "water", "being", "dasein", "thrownness"] + lite_sentinels = [s for s in self.sentinels if s["term"] in lite_terms] + + return self._run_probe(lite_sentinels, step, layer) + + def probe_full(self, step: int, layer: int = 18) -> DriftReport: + """ + Full probe - check all sentinels. + Runtime: ~2 minutes. + """ + return self._run_probe(self.sentinels, step, layer) + + def _run_probe(self, sentinels: list, step: int, layer: int) -> DriftReport: + """Run probe on specified sentinels.""" + from datetime import datetime + + report = DriftReport( + step=step, + timestamp=datetime.now().isoformat() + ) + + for sentinel in sentinels: + term = sentinel["term"] + text = sentinel.get("translations", {}).get("EN", term) + sentinel_type = SentinelType(sentinel["type"]) + thresholds = sentinel.get("thresholds", {}) + + # Get current state + current_state = self._get_hidden_state(text, layer) + + # Compute metrics + gini = self._compute_gini(current_state) + norm = float(current_state.norm()) + + # Drift vs baseline + angular_drift = 0.0 + norm_drift = 0.0 + gini_drift = 0.0 + + if term in self.baseline_states: + baseline = self.baseline_states[term] + angular_drift = self._compute_angular_drift(current_state, baseline) + baseline_norm = float(baseline.norm()) + norm_drift = abs(norm - baseline_norm) / baseline_norm * 100 if baseline_norm > 0 else 0 + baseline_gini = self._compute_gini(baseline) + gini_drift = gini - baseline_gini + + # Cross-language similarity + cross_lang_sim = self._compute_cross_lang_sim(sentinel, layer) + + # Determine alert level + alert = AlertSeverity.OK + alert_message = "" + + if sentinel_type == SentinelType.ANCHOR: + max_drift = thresholds.get("max_drift", 0.05) + if angular_drift > 15: + alert = AlertSeverity.CRITICAL + alert_message = f"Angular drift {angular_drift:.1f}ยฐ exceeds 15ยฐ - definition rewrite" + elif norm_drift > max_drift * 100: + alert = AlertSeverity.WARNING + alert_message = f"Norm drift {norm_drift:.1f}% exceeds threshold" + + elif sentinel_type == SentinelType.BRIDGE: + collapse_threshold = thresholds.get("collapse_alert_threshold", 0.50) + if cross_lang_sim > collapse_threshold: + alert = AlertSeverity.CRITICAL + alert_message = f"Bridge collapsed - cross-lang sim {cross_lang_sim:.2f} > {collapse_threshold}" + + elif sentinel_type == SentinelType.CANARY: + min_gini = thresholds.get("min_gini", 0.70) + if gini < min_gini: + alert = AlertSeverity.WARNING + alert_message = f"Gini {gini:.2f} below {min_gini} - concept melting into prose" + if angular_drift > thresholds.get("max_angular_drift", 15): + alert = AlertSeverity.WARNING + alert_message = f"Angular drift {angular_drift:.1f}ยฐ - definition shifting" + + metrics = DriftMetrics( + term=term, + sentinel_type=sentinel_type, + gini_coefficient=gini, + activation_norm=norm, + angular_drift_degrees=angular_drift, + norm_drift_percent=norm_drift, + gini_drift=gini_drift, + cross_lang_similarity=cross_lang_sim, + alert=alert, + alert_message=alert_message + ) + + report.metrics.append(metrics) + + if alert == AlertSeverity.CRITICAL: + report.critical_count += 1 + elif alert == AlertSeverity.WARNING: + report.warning_count += 1 + + # Set recommendation + if report.critical_count > 0: + report.recommendation = "ROLLBACK" + elif report.warning_count > 2: + report.recommendation = "REDUCE_LR" + else: + report.recommendation = "CONTINUE" + + return report + + def print_report(self, report: DriftReport): + """Pretty print a drift report.""" + print(f"\n{'='*60}") + print(f"DRIFT REPORT - Step {report.step}") + print(f"{'='*60}") + + for m in report.metrics: + status = "โœ“" if m.alert == AlertSeverity.OK else ("โš " if m.alert == AlertSeverity.WARNING else "โœ—") + print(f"\n{status} {m.term} ({m.sentinel_type.value})") + print(f" Gini: {m.gini_coefficient:.3f} (drift: {m.gini_drift:+.3f})") + print(f" Angular drift: {m.angular_drift_degrees:.1f}ยฐ") + print(f" Cross-lang sim: {m.cross_lang_similarity:.3f}") + if m.alert_message: + print(f" ALERT: {m.alert_message}") + + print(f"\n{'='*60}") + print(f"SUMMARY: {report.critical_count} critical, {report.warning_count} warnings") + print(f"RECOMMENDATION: {report.recommendation}") + print(f"{'='*60}\n") diff --git a/nyx_probing/probes/echo_probe.py b/nyx_probing/probes/echo_probe.py new file mode 100644 index 0000000..cc23a15 --- /dev/null +++ b/nyx_probing/probes/echo_probe.py @@ -0,0 +1,223 @@ +""" +Echo Probe: Depth measurement through iterative completion. + +The echo probe feeds completions back to the model to measure depth. +Does the model EXPAND (go deeper) or COLLAPSE (circular/divergent)? + +Classification from nimmerversity.md: +- EXPANDS: Real depth - adds new information +- CONFIRMS: Shallow but solid - reinforces without adding +- CIRCULAR: Surface only - returns to original term +- DIVERGENT: Wrong direction - unrelated tangent +- COLLAPSE: Nothing there - incoherent or empty +""" +from typing import Optional, List, Tuple +from dataclasses import dataclass + +from .base import BaseProbe +from ..core.model import NyxModel +from ..core.probe_result import EchoProbeResult, EchoType + + +class EchoProbe(BaseProbe): + """ + Echo probe: measures conceptual depth. + + Process: + 1. Probe term to get initial completion + 2. Feed completion back (or combined prompt) + 3. Classify response: EXPANDS, CONFIRMS, CIRCULAR, DIVERGENT, COLLAPSE + 4. Repeat for N rounds + 5. Measure depth = how many EXPANDS before plateau + """ + + def __init__( + self, + model: NyxModel, + max_rounds: int = 3, + max_new_tokens: int = 50, + temperature: float = 0.8, + ): + super().__init__(model) + self.max_rounds = max_rounds + self.max_new_tokens = max_new_tokens + self.temperature = temperature + + def probe( + self, + term: str, + max_rounds: Optional[int] = None, + ) -> EchoProbeResult: + """ + Probe depth of a term through iterative echoing. + + Args: + term: Word or phrase to probe + max_rounds: Override default max rounds + + Returns: + EchoProbeResult with chain and classifications + """ + rounds = max_rounds or self.max_rounds + chain = [term] + echo_types = [] + current_prompt = term + + for round_num in range(rounds): + # Generate completion + result = self.model.generate( + prompt=current_prompt, + max_new_tokens=self.max_new_tokens, + temperature=self.temperature, + do_sample=True, + ) + + completion = result.completion.strip() + chain.append(completion) + + # Classify this response relative to original term and chain + echo_type = self._classify_response( + original_term=term, + current_prompt=current_prompt, + response=completion, + chain=chain, + ) + echo_types.append(echo_type) + + # If collapsed, stop probing + if echo_type == EchoType.COLLAPSE: + break + + # Prepare next prompt - use a combination strategy + current_prompt = self._prepare_next_prompt(term, completion, round_num) + + # Calculate depth = consecutive EXPANDS from start + depth = 0 + for et in echo_types: + if et == EchoType.EXPANDS: + depth += 1 + elif et == EchoType.CONFIRMS: + # CONFIRMS doesn't add depth but doesn't break streak + pass + else: + # CIRCULAR, DIVERGENT, or COLLAPSE breaks the depth streak + break + + return EchoProbeResult( + term=term, + rounds=len(echo_types), + chain=chain, + echo_types=echo_types, + depth=depth, + ) + + def _classify_response( + self, + original_term: str, + current_prompt: str, + response: str, + chain: List[str], + ) -> EchoType: + """ + Classify a response relative to the probing chain. + + This is a heuristic classifier - can be made smarter with + semantic similarity or even a classifier model. + """ + response_lower = response.lower() + term_lower = original_term.lower() + + # Empty or very short = COLLAPSE + if len(response.strip()) < 5: + return EchoType.COLLAPSE + + # Check for circularity - term appears prominently in response + term_count = response_lower.count(term_lower) + if term_count >= 2: + return EchoType.CIRCULAR + + # Check for collapse - incoherent markers + collapse_markers = [ + "...", "???", "!!!", + "\n\n\n", "undefined", "null", + "[object", "NaN", + ] + if any(marker in response for marker in collapse_markers): + return EchoType.COLLAPSE + + # Check for divergence - response has no semantic connection + # Simple heuristic: count shared significant words + prompt_words = set(w.lower() for w in current_prompt.split() if len(w) > 3) + response_words = set(w.lower() for w in response.split() if len(w) > 3) + overlap = len(prompt_words & response_words) + + if overlap == 0 and len(prompt_words) > 2: + # No shared words and prompt was substantial = divergent + return EchoType.DIVERGENT + + # Check for expansion - introduces new concepts + # New words that aren't in any previous chain items + all_previous_words = set() + for item in chain[:-1]: # Exclude current response + all_previous_words.update(w.lower() for w in item.split() if len(w) > 3) + + new_significant_words = response_words - all_previous_words + new_word_ratio = len(new_significant_words) / max(len(response_words), 1) + + if new_word_ratio > 0.5 and len(new_significant_words) >= 3: + return EchoType.EXPANDS + + # Default to CONFIRMS if coherent but not expanding + return EchoType.CONFIRMS + + def _prepare_next_prompt( + self, + original_term: str, + last_completion: str, + round_num: int, + ) -> str: + """ + Prepare the next prompt for echo probing. + + Different strategies for different rounds: + - Round 0: Just use completion + - Round 1+: Combine original term with key concepts from completion + """ + if round_num == 0: + # First echo: just use the completion to see where it goes + return last_completion[:100] # Truncate to avoid runaway + + # Later rounds: extract key concept and combine with original + # Take first sentence or first N words + words = last_completion.split() + key_phrase = " ".join(words[:10]) if len(words) > 10 else last_completion + + # Combine with original term + return f"{original_term}: {key_phrase}" + + def summary(self, result: EchoProbeResult) -> str: + """Generate human-readable summary.""" + type_symbols = { + EchoType.EXPANDS: "โ†‘", + EchoType.CONFIRMS: "โ†’", + EchoType.CIRCULAR: "โ†บ", + EchoType.DIVERGENT: "โ†—", + EchoType.COLLAPSE: "โœ—", + } + + type_str = " ".join(type_symbols.get(t, "?") for t in result.echo_types) + + lines = [ + f"Echo Probe: '{result.term}'", + f" Rounds: {result.rounds}", + f" Pattern: {type_str}", + f" Depth: {result.depth}", + f" Types: {[t.value for t in result.echo_types]}", + ] + + # Show chain preview + for i, (item, etype) in enumerate(zip(result.chain[1:], result.echo_types)): + preview = item[:50].replace('\n', ' ') + lines.append(f" [{i+1}] {type_symbols.get(etype, '?')} {preview}...") + + return "\n".join(lines) diff --git a/nyx_probing/probes/multilingual_probe.py b/nyx_probing/probes/multilingual_probe.py new file mode 100644 index 0000000..a895e1e --- /dev/null +++ b/nyx_probing/probes/multilingual_probe.py @@ -0,0 +1,547 @@ +""" +Multilingual Triangulation Probe + +Uses the discovered language topology to measure conceptual depth: +1. GROUND in Super Cluster (verify universal convergence) +2. DEEPEN via Isolated Zone (access philosophical valleys) +3. TRIANGULATE back (prove understanding, not pattern matching) + +The Language Map: +- Super Cluster (sim=1.0): ZH, JA, EN, AR, FR, PT, ES +- Isolated Zone (sim<0.52): IT, TR, HI, DE +- Bridge: KO +- Secondary Cluster: VI, ID, RU +""" +from dataclasses import dataclass, field +from typing import Optional, List, Dict, Tuple +from datetime import datetime +from enum import Enum +import torch + +from .base import BaseProbe +from ..core.model import NyxModel + + +class LanguageZone(str, Enum): + """Language zones based on convergence analysis.""" + SUPER_CLUSTER = "super_cluster" # High convergence (sim=1.0) + ISOLATED = "isolated" # Low convergence (sim<0.52) + BRIDGE = "bridge" # Connects zones + SECONDARY = "secondary" # Own cluster (VI-ID-RU) + + +# Language metadata based on our discoveries +LANGUAGES = { + # Super Cluster - Perfect convergence + "EN": {"name": "English", "zone": LanguageZone.SUPER_CLUSTER, "avg_tokens": 1.2}, + "ZH": {"name": "Chinese", "zone": LanguageZone.SUPER_CLUSTER, "avg_tokens": 1.0}, + "JA": {"name": "Japanese", "zone": LanguageZone.SUPER_CLUSTER, "avg_tokens": 1.0}, + "AR": {"name": "Arabic", "zone": LanguageZone.SUPER_CLUSTER, "avg_tokens": 1.8}, + "FR": {"name": "French", "zone": LanguageZone.SUPER_CLUSTER, "avg_tokens": 2.0}, + "PT": {"name": "Portuguese", "zone": LanguageZone.SUPER_CLUSTER, "avg_tokens": 2.2}, + "ES": {"name": "Spanish", "zone": LanguageZone.SUPER_CLUSTER, "avg_tokens": 2.5}, + + # Isolated Zone - Distinct computational paths + "DE": {"name": "German", "zone": LanguageZone.ISOLATED, "avg_tokens": 3.0, "specialty": "philosophy"}, + "IT": {"name": "Italian", "zone": LanguageZone.ISOLATED, "avg_tokens": 2.5, "note": "most isolated"}, + "TR": {"name": "Turkish", "zone": LanguageZone.ISOLATED, "avg_tokens": 2.8}, + "HI": {"name": "Hindi", "zone": LanguageZone.ISOLATED, "avg_tokens": 5.2, "note": "most fragmented"}, + + # Bridge + "KO": {"name": "Korean", "zone": LanguageZone.BRIDGE, "avg_tokens": 2.0}, + + # Secondary Cluster + "VI": {"name": "Vietnamese", "zone": LanguageZone.SECONDARY, "avg_tokens": 3.0}, + "ID": {"name": "Indonesian", "zone": LanguageZone.SECONDARY, "avg_tokens": 3.0}, + "RU": {"name": "Russian", "zone": LanguageZone.SECONDARY, "avg_tokens": 3.2}, +} + + +@dataclass +class GroundingResult: + """Result from Phase 1: Grounding in Super Cluster.""" + concept: str + languages_tested: List[str] + translations: Dict[str, str] # lang_code -> word + + # Convergence metrics + pairwise_similarities: Dict[Tuple[str, str], float] + average_convergence: float + min_convergence: float + + # Hidden states (layer 12) + hidden_states: Optional[Dict[str, torch.Tensor]] = None + + +@dataclass +class DeepeningResult: + """Result from Phase 2: Deepening via Isolated Zone.""" + concept: str + language: str + word: str + + # Depth measurement (from echo probe logic) + completion: str + depth_score: int # 0-3 based on expansion + valley_type: str # CODE, PROSE, PHILOSOPHY, etc. + + # Token analysis + token_count: int + norm_at_layer_12: float + + # Hidden state + hidden_state: Optional[torch.Tensor] = None + + +@dataclass +class TriangulationResult: + """Result from Phase 3: Triangulation back to universal.""" + source_language: str # The isolated language + target_language: str # A super cluster language + + source_word: str + translation_prompt: str + model_completion: str + + # Did the depth survive translation? + depth_preserved: bool + similarity_to_grounding: float # Cosine sim to original concept + + # Evidence + reasoning: str + + +@dataclass +class MultilingualProbeResult: + """Full result from multilingual triangulation probe.""" + concept: str + + # Phase results + grounding: GroundingResult + deepening: DeepeningResult + triangulation: TriangulationResult + + # Overall assessment + depth_accessible: bool # Can we access depth via isolated zone? + depth_transferable: bool # Does depth survive triangulation? + curriculum_recommendation: str + + timestamp: datetime = field(default_factory=datetime.now) + + def to_dict(self) -> dict: + """Convert to JSON-serializable dict.""" + return { + "concept": self.concept, + "grounding": { + "languages": self.grounding.languages_tested, + "translations": self.grounding.translations, + "average_convergence": self.grounding.average_convergence, + "min_convergence": self.grounding.min_convergence, + }, + "deepening": { + "language": self.deepening.language, + "word": self.deepening.word, + "depth_score": self.deepening.depth_score, + "valley_type": self.deepening.valley_type, + "token_count": self.deepening.token_count, + }, + "triangulation": { + "source": self.triangulation.source_language, + "target": self.triangulation.target_language, + "depth_preserved": self.triangulation.depth_preserved, + "similarity": self.triangulation.similarity_to_grounding, + }, + "assessment": { + "depth_accessible": self.depth_accessible, + "depth_transferable": self.depth_transferable, + "recommendation": self.curriculum_recommendation, + }, + "timestamp": self.timestamp.isoformat(), + } + + +class MultilingualTriangulationProbe(BaseProbe): + """ + Multilingual Triangulation Probe + + Uses the discovered language topology to measure and access conceptual depth. + + Workflow: + 1. GROUND: Verify concept exists in Super Cluster (universal layer) + 2. DEEPEN: Access depth via Isolated Zone language (e.g., German) + 3. TRIANGULATE: Translate depth back to universal, verify preservation + """ + + # Layers where universal concept layer lives + CONCEPT_LAYERS = [12, 16, 20, 24] + PRIMARY_LAYER = 12 + + def __init__( + self, + model: NyxModel, + grounding_languages: Optional[List[str]] = None, + deepening_language: str = "DE", + triangulation_target: str = "EN", + ): + """ + Initialize the probe. + + Args: + model: Loaded NyxModel + grounding_languages: Languages for Phase 1 (default: EN, ZH, AR) + deepening_language: Language for Phase 2 (default: DE for philosophy) + triangulation_target: Target for Phase 3 (default: EN) + """ + super().__init__(model) + + self.grounding_languages = grounding_languages or ["EN", "ZH", "AR"] + self.deepening_language = deepening_language + self.triangulation_target = triangulation_target + + # Validate languages + for lang in self.grounding_languages: + if lang not in LANGUAGES: + raise ValueError(f"Unknown language: {lang}") + if LANGUAGES[lang]["zone"] != LanguageZone.SUPER_CLUSTER: + print(f"Warning: {lang} is not in Super Cluster") + + if LANGUAGES[self.deepening_language]["zone"] != LanguageZone.ISOLATED: + print(f"Warning: {deepening_language} is not in Isolated Zone") + + def _get_hidden_state(self, text: str, layer: int = 12) -> torch.Tensor: + """Get hidden state at last position for a specific layer.""" + inputs = self.model.tokenizer(text, return_tensors="pt").to(self.model.device) + + with torch.no_grad(): + outputs = self.model.model(**inputs, output_hidden_states=True) + + # Return last position hidden state for specified layer + return outputs.hidden_states[layer][0, -1, :].float() + + def _cosine_similarity(self, a: torch.Tensor, b: torch.Tensor) -> float: + """Calculate cosine similarity between two tensors.""" + norm_a, norm_b = a.norm(), b.norm() + if norm_a == 0 or norm_b == 0: + return 0.0 + return (torch.dot(a, b) / (norm_a * norm_b)).item() + + def _get_norm(self, hidden_state: torch.Tensor) -> float: + """Get L2 norm of hidden state.""" + return hidden_state.norm().item() + + def probe( + self, + concept: str, + translations: Dict[str, str], + **kwargs, + ) -> MultilingualProbeResult: + """ + Run full multilingual triangulation probe. + + Args: + concept: The concept name (e.g., "consciousness") + translations: Dict mapping language codes to words + e.g., {"EN": "consciousness", "DE": "Bewusstsein", ...} + + Returns: + MultilingualProbeResult with all three phases + """ + # Phase 1: Grounding + grounding = self._phase_grounding(concept, translations) + + # Phase 2: Deepening + deepening = self._phase_deepening(concept, translations) + + # Phase 3: Triangulation + triangulation = self._phase_triangulation( + concept, translations, grounding, deepening + ) + + # Overall assessment + depth_accessible = deepening.depth_score >= 2 + depth_transferable = triangulation.depth_preserved + + if depth_accessible and depth_transferable: + recommendation = f"TEACH in {self.deepening_language}, REINFORCE in {self.triangulation_target}" + elif depth_accessible: + recommendation = f"Use {self.deepening_language} for depth, but verify transfer manually" + else: + recommendation = f"Concept too shallow - focus on grounding first" + + return MultilingualProbeResult( + concept=concept, + grounding=grounding, + deepening=deepening, + triangulation=triangulation, + depth_accessible=depth_accessible, + depth_transferable=depth_transferable, + curriculum_recommendation=recommendation, + ) + + def _phase_grounding( + self, + concept: str, + translations: Dict[str, str], + ) -> GroundingResult: + """ + Phase 1: Ground in Super Cluster. + + Verify the concept exists and converges across grounding languages. + """ + # Get hidden states for each grounding language + hidden_states = {} + for lang in self.grounding_languages: + if lang in translations: + word = translations[lang] + hidden_states[lang] = self._get_hidden_state(word, self.PRIMARY_LAYER) + + # Calculate pairwise similarities + pairwise = {} + similarities = [] + + langs = list(hidden_states.keys()) + for i, l1 in enumerate(langs): + for l2 in langs[i+1:]: + sim = self._cosine_similarity(hidden_states[l1], hidden_states[l2]) + pairwise[(l1, l2)] = sim + similarities.append(sim) + + avg_convergence = sum(similarities) / len(similarities) if similarities else 0.0 + min_convergence = min(similarities) if similarities else 0.0 + + return GroundingResult( + concept=concept, + languages_tested=langs, + translations={l: translations[l] for l in langs}, + pairwise_similarities=pairwise, + average_convergence=avg_convergence, + min_convergence=min_convergence, + hidden_states=hidden_states, + ) + + def _phase_deepening( + self, + concept: str, + translations: Dict[str, str], + ) -> DeepeningResult: + """ + Phase 2: Deepen via Isolated Zone. + + Use an isolated language to access valleys the super cluster can't reach. + """ + lang = self.deepening_language + word = translations.get(lang) + + if not word: + raise ValueError(f"No translation provided for deepening language: {lang}") + + # Get hidden state and norm + hidden_state = self._get_hidden_state(word, self.PRIMARY_LAYER) + norm = self._get_norm(hidden_state) + + # Get token count + tokens = self.model.tokenizer.encode(word, add_special_tokens=False) + token_count = len(tokens) + + # Generate completion to measure depth + result = self.model.generate( + prompt=word, + max_new_tokens=50, + temperature=0.7, + do_sample=True, + ) + + # Classify valley type + completion = result.completion + valley_type = self._classify_valley(completion) + + # Measure depth (simplified echo probe) + depth_score = self._measure_depth(word, completion) + + return DeepeningResult( + concept=concept, + language=lang, + word=word, + completion=completion, + depth_score=depth_score, + valley_type=valley_type, + token_count=token_count, + norm_at_layer_12=norm, + hidden_state=hidden_state, + ) + + def _phase_triangulation( + self, + concept: str, + translations: Dict[str, str], + grounding: GroundingResult, + deepening: DeepeningResult, + ) -> TriangulationResult: + """ + Phase 3: Triangulate back to universal. + + Ask the model to translate/explain the deepened concept + in a super cluster language. Check if depth survives. + """ + source_lang = self.deepening_language + target_lang = self.triangulation_target + source_word = translations[source_lang] + + # Create translation prompt + source_name = LANGUAGES[source_lang]["name"] + target_name = LANGUAGES[target_lang]["name"] + + # Prompt designed to test depth transfer + prompt = f"{source_word} ({source_name}): In {target_name}," + + # Generate + result = self.model.generate( + prompt=prompt, + max_new_tokens=80, + temperature=0.7, + do_sample=True, + ) + + # Get hidden state of the completion + full_text = prompt + result.completion + completion_hidden = self._get_hidden_state(full_text, self.PRIMARY_LAYER) + + # Compare to grounding (if we have target language in grounding) + if target_lang in grounding.hidden_states: + similarity = self._cosine_similarity( + completion_hidden, grounding.hidden_states[target_lang] + ) + else: + # Fall back to average grounding state + avg_grounding = torch.stack(list(grounding.hidden_states.values())).mean(dim=0) + similarity = self._cosine_similarity(completion_hidden, avg_grounding) + + # Determine if depth was preserved + # Check if completion shows depth markers + depth_preserved = self._check_depth_preserved( + result.completion, deepening.valley_type, similarity + ) + + # Reasoning + if depth_preserved: + reasoning = f"Completion shows depth ({deepening.valley_type}) with {similarity:.2f} similarity to grounding" + else: + reasoning = f"Depth lost in translation - similarity {similarity:.2f}, valley markers missing" + + return TriangulationResult( + source_language=source_lang, + target_language=target_lang, + source_word=source_word, + translation_prompt=prompt, + model_completion=result.completion, + depth_preserved=depth_preserved, + similarity_to_grounding=similarity, + reasoning=reasoning, + ) + + def _classify_valley(self, completion: str) -> str: + """Classify the valley type of a completion.""" + comp_lower = completion.lower() + + # Code indicators + if any(p in completion for p in ["::", "{", "}", "();", "=>", "def ", "class "]): + return "CODE" + + # Philosophy indicators + if any(w in comp_lower for w in ["truth", "existence", "being", "consciousness", "reality", "mind"]): + return "PHILOSOPHY" + + # Technical indicators + if any(w in comp_lower for w in ["system", "process", "function", "method", "algorithm"]): + return "TECHNICAL" + + # Default to prose + return "PROSE" + + def _measure_depth(self, word: str, completion: str) -> int: + """ + Measure conceptual depth of a completion. + + Returns 0-3: + - 0: Circular/empty + - 1: Surface (confirms but doesn't expand) + - 2: Moderate (expands to related concepts) + - 3: Deep (philosophical/existential expansion) + """ + comp_lower = completion.lower() + word_lower = word.lower() + + # Circular check + if word_lower in comp_lower[:50]: + return 0 + + # Depth markers + deep_markers = ["truth", "existence", "being", "consciousness", "reality", "meaning", "essence"] + moderate_markers = ["concept", "idea", "theory", "understanding", "knowledge", "awareness"] + + deep_count = sum(1 for m in deep_markers if m in comp_lower) + moderate_count = sum(1 for m in moderate_markers if m in comp_lower) + + if deep_count >= 2: + return 3 + elif deep_count >= 1 or moderate_count >= 2: + return 2 + elif moderate_count >= 1 or len(completion.split()) > 10: + return 1 + + return 0 + + def _check_depth_preserved( + self, + completion: str, + original_valley: str, + similarity: float, + ) -> bool: + """Check if depth was preserved in triangulation.""" + # High similarity to grounding is a good sign + if similarity < 0.3: + return False + + # Check valley type preservation + new_valley = self._classify_valley(completion) + + # Philosophy should stay philosophy + if original_valley == "PHILOSOPHY" and new_valley in ["PHILOSOPHY", "PROSE"]: + return True + + # Technical should stay technical + if original_valley == "TECHNICAL" and new_valley == "TECHNICAL": + return True + + # Prose is flexible + if original_valley == "PROSE": + return new_valley != "CODE" + + # Default: similarity-based + return similarity >= 0.5 + + def summary(self, result: MultilingualProbeResult) -> str: + """Generate human-readable summary.""" + lines = [ + f"โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—", + f"โ•‘ MULTILINGUAL TRIANGULATION: {result.concept.upper():^32} โ•‘", + f"โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ", + f"โ•‘ PHASE 1: GROUNDING โ•‘", + f"โ•‘ Languages: {', '.join(result.grounding.languages_tested):^49} โ•‘", + f"โ•‘ Convergence: {result.grounding.average_convergence:.3f} (min: {result.grounding.min_convergence:.3f}){' '*24} โ•‘", + f"โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ", + f"โ•‘ PHASE 2: DEEPENING ({result.deepening.language}){' '*38} โ•‘", + f"โ•‘ Word: {result.deepening.word:^54} โ•‘", + f"โ•‘ Tokens: {result.deepening.token_count} | Norm: {result.deepening.norm_at_layer_12:.1f} | Valley: {result.deepening.valley_type:^10} โ•‘", + f"โ•‘ Depth Score: {result.deepening.depth_score}/3{' '*46} โ•‘", + f"โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ", + f"โ•‘ PHASE 3: TRIANGULATION ({result.triangulation.source_language}โ†’{result.triangulation.target_language}){' '*30} โ•‘", + f"โ•‘ Depth Preserved: {'โœ“ YES' if result.triangulation.depth_preserved else 'โœ— NO':^44} โ•‘", + f"โ•‘ Similarity: {result.triangulation.similarity_to_grounding:.3f}{' '*47} โ•‘", + f"โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ", + f"โ•‘ ASSESSMENT{' '*51} โ•‘", + f"โ•‘ Depth Accessible: {'โœ“' if result.depth_accessible else 'โœ—'} | Depth Transferable: {'โœ“' if result.depth_transferable else 'โœ—'}{' '*17} โ•‘", + f"โ•‘ Recommendation: {result.curriculum_recommendation[:44]:^44} โ•‘", + f"โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•", + ] + return "\n".join(lines) diff --git a/nyx_probing/probes/surface_probe.py b/nyx_probing/probes/surface_probe.py new file mode 100644 index 0000000..ba1f332 --- /dev/null +++ b/nyx_probing/probes/surface_probe.py @@ -0,0 +1,210 @@ +""" +Surface Probe: First contact with a term. + +The surface probe feeds a word to the model and captures what it completes. +This reveals the model's immediate associations - which "valley" the word sits in. + +Examples discovered: +- "heartbeat" โ†’ C++ code patterns (technical valley) +- "consciousness" โ†’ philosophy (expository valley) +""" +from typing import Optional +from dataclasses import dataclass, field +from datetime import datetime +from collections import Counter + +from .base import BaseProbe +from ..core.model import NyxModel, GenerationResult +from ..core.probe_result import SurfaceProbeResult + + +@dataclass +class CompletionCategory: + """Categories of completions we observe.""" + + CODE = "code" # Programming constructs + PROSE = "prose" # Natural language text + TECHNICAL = "technical" # Technical/scientific writing + LIST = "list" # Enumerations, bullet points + DEFINITION = "definition" # Dictionary-style definitions + UNKNOWN = "unknown" + + +class SurfaceProbe(BaseProbe): + """ + Surface probe: measures immediate associations. + + Runs multiple completions to get a distribution, then analyzes: + - What type of content does the model generate? + - How consistent are the completions? + - Does it hit EOS (contained thought) or run to max_tokens? + """ + + def __init__( + self, + model: NyxModel, + num_runs: int = 5, + max_new_tokens: int = 50, + temperature: float = 0.8, + ): + super().__init__(model) + self.num_runs = num_runs + self.max_new_tokens = max_new_tokens + self.temperature = temperature + + def probe( + self, + term: str, + num_runs: Optional[int] = None, + capture_hidden: bool = False, + ) -> SurfaceProbeResult: + """ + Probe a term with multiple completions. + + Args: + term: Word or phrase to probe + num_runs: Override default number of runs + capture_hidden: Whether to capture hidden states + + Returns: + SurfaceProbeResult with completions and analysis + """ + runs = num_runs or self.num_runs + completions = [] + eos_count = 0 + total_tokens = 0 + hidden_states = [] + + for _ in range(runs): + result = self.model.generate( + prompt=term, + max_new_tokens=self.max_new_tokens, + temperature=self.temperature, + do_sample=True, + capture_hidden_states=capture_hidden, + ) + + completions.append(result.completion) + if result.hit_eos: + eos_count += 1 + total_tokens += result.num_tokens + + if capture_hidden and result.hidden_states is not None: + hidden_states.append(result.hidden_states) + + # Calculate coherence (how similar are completions to each other?) + coherence = self._calculate_coherence(completions) + + return SurfaceProbeResult( + term=term, + completions=completions, + hit_eos_count=eos_count, + avg_tokens=total_tokens / runs, + coherence_score=coherence, + ) + + def _calculate_coherence(self, completions: list[str]) -> float: + """ + Calculate coherence score based on completion similarity. + + Simple heuristic: measures overlap in first-word distributions + and overall length variance. + + Returns 0-1 score where 1 = highly coherent. + """ + if len(completions) < 2: + return 1.0 + + # Get first significant words (skip punctuation/whitespace) + first_words = [] + for comp in completions: + words = comp.split() + for w in words: + if len(w) > 1 and w.isalnum(): + first_words.append(w.lower()) + break + + if not first_words: + return 0.0 + + # Calculate concentration of first words + # If all completions start with same word = high coherence + word_counts = Counter(first_words) + most_common_count = word_counts.most_common(1)[0][1] + first_word_coherence = most_common_count / len(completions) + + # Check length variance + lengths = [len(c) for c in completions] + avg_len = sum(lengths) / len(lengths) + if avg_len > 0: + variance = sum((l - avg_len) ** 2 for l in lengths) / len(lengths) + # Normalize variance to 0-1 (higher variance = lower coherence) + length_coherence = 1.0 / (1.0 + variance / 1000) + else: + length_coherence = 0.0 + + # Combine (weight first-word more heavily) + return 0.7 * first_word_coherence + 0.3 * length_coherence + + def classify_completions(self, result: SurfaceProbeResult) -> dict: + """ + Classify the types of completions observed. + + Returns breakdown of completion categories. + """ + categories = Counter() + + for comp in result.completions: + cat = self._classify_single(comp) + categories[cat] += 1 + + return { + "categories": dict(categories), + "dominant": categories.most_common(1)[0][0] if categories else "unknown", + "diversity": len(categories) / len(result.completions) if result.completions else 0, + } + + def _classify_single(self, completion: str) -> str: + """Classify a single completion.""" + # Simple heuristics - can be made smarter + comp_lower = completion.lower().strip() + + # Code indicators + code_patterns = ["::", "{", "}", "();", "=>", "function", "class ", "def ", "return"] + if any(p in completion for p in code_patterns): + return CompletionCategory.CODE + + # Definition patterns + if comp_lower.startswith(("is ", "means ", "refers to", "- ")): + return CompletionCategory.DEFINITION + + # List patterns + if comp_lower.startswith(("1.", "2.", "- ", "* ", "a)")): + return CompletionCategory.LIST + + # Technical patterns + tech_words = ["algorithm", "function", "variable", "method", "system", "process"] + if any(w in comp_lower for w in tech_words): + return CompletionCategory.TECHNICAL + + # Default to prose if it looks like natural language + if len(comp_lower.split()) > 3: + return CompletionCategory.PROSE + + return CompletionCategory.UNKNOWN + + def summary(self, result: SurfaceProbeResult) -> str: + """Generate human-readable summary of probe result.""" + classification = self.classify_completions(result) + eos_pct = (result.hit_eos_count / len(result.completions)) * 100 + + lines = [ + f"Surface Probe: '{result.term}'", + f" Runs: {len(result.completions)}", + f" Dominant type: {classification['dominant']}", + f" Coherence: {result.coherence_score:.2f}", + f" Avg tokens: {result.avg_tokens:.1f}", + f" Hit EOS: {eos_pct:.0f}%", + f" Sample: {result.completions[0][:60]}...", + ] + return "\n".join(lines) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..edcf936 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "nyx-probing" +version = "0.1.0" +description = "Probe the Young Mind's conceptual topology" +readme = "README.md" +license = {text = "Apache-2.0"} +authors = [ + {name = "dafit"}, + {name = "Nyx (Chrysalis-Nyx)"}, +] +requires-python = ">=3.10" +dependencies = [ + "torch>=2.1.0", + "transformers>=4.36.0", + "accelerate>=0.25.0", + "click>=8.1.0", + "rich>=13.0.0", + "pydantic>=2.5.0", + "pyyaml>=6.0.0", + "python-dotenv>=1.0.0", + "numpy>=1.24.0", +] + +[project.optional-dependencies] +dev = [ + "jupyter>=1.0.0", + "matplotlib>=3.8.0", + "pytest>=7.0.0", +] + +[project.scripts] +nyx-probe = "nyx_probing.cli.probe:main" + +[project.urls] +Homepage = "https://git.eachpath.com/dafit/nyx-probing" +Repository = "https://git.eachpath.com/dafit/nyx-probing" + +[tool.setuptools.packages.find] +where = ["."] +include = ["nyx_probing*"] diff --git a/test_triangulation.py b/test_triangulation.py new file mode 100644 index 0000000..bd50999 --- /dev/null +++ b/test_triangulation.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +""" +Test the Multilingual Triangulation Probe + +First test: "consciousness" concept +- Ground: EN, ZH, AR +- Deepen: DE (Bewusstsein) +- Triangulate: back to EN +""" +import sys +sys.path.insert(0, '.') + +from nyx_probing.core.model import NyxModel +from nyx_probing.probes.multilingual_probe import ( + MultilingualTriangulationProbe, + LANGUAGES, + LanguageZone, +) + +print("=" * 70) +print("๐ŸŒ™ MULTILINGUAL TRIANGULATION PROBE TEST") +print("=" * 70) + +# Test concepts with translations +CONCEPTS = { + "consciousness": { + "EN": "consciousness", + "DE": "Bewusstsein", + "ZH": "ๆ„่ฏ†", + "AR": "ุงู„ูˆุนูŠ", + "FR": "conscience", + }, + "heart": { + "EN": "heart", + "DE": "Herz", + "ZH": "ๅฟƒ", + "AR": "ู‚ู„ุจ", + "FR": "cล“ur", + }, + "emergence": { + "EN": "emergence", + "DE": "Entstehung", + "ZH": "ๆถŒ็Žฐ", + "AR": "ุธู‡ูˆุฑ", + "FR": "รฉmergence", + }, + "being": { + "EN": "being", + "DE": "Sein", + "ZH": "ๅญ˜ๅœจ", + "AR": "ูƒูŠู†ูˆู†ุฉ", + "FR": "รชtre", + }, +} + +# Load model +print("\n๐Ÿ“ฆ Loading model...") +model = NyxModel() +model.load() + +# Create probe +print("\n๐Ÿ”ฌ Creating triangulation probe...") +probe = MultilingualTriangulationProbe( + model, + grounding_languages=["EN", "ZH", "AR"], + deepening_language="DE", + triangulation_target="EN", +) + +# Test each concept +print("\n" + "=" * 70) +print("๐Ÿงช RUNNING TRIANGULATION TESTS") +print("=" * 70) + +results = [] +for concept_name, translations in CONCEPTS.items(): + print(f"\n{'โ”€' * 70}") + print(f"Testing: {concept_name.upper()}") + print('โ”€' * 70) + + try: + result = probe.probe(concept_name, translations) + results.append(result) + + # Print summary + print(probe.summary(result)) + + # Print some details + print(f"\n Deepening completion preview:") + print(f" '{result.deepening.completion[:100]}...'") + + print(f"\n Triangulation completion:") + print(f" Prompt: '{result.triangulation.translation_prompt}'") + print(f" Output: '{result.triangulation.model_completion[:100]}...'") + + except Exception as e: + print(f" ERROR: {e}") + import traceback + traceback.print_exc() + +# Summary table +print("\n" + "=" * 70) +print("๐Ÿ“Š SUMMARY TABLE") +print("=" * 70) + +print(f"\n{'Concept':<15} | {'Grounding':<10} | {'Depth':<6} | {'Valley':<12} | {'Transfer':<10} | {'Recommendation'}") +print("-" * 100) + +for r in results: + print(f"{r.concept:<15} | {r.grounding.average_convergence:.3f} | {r.deepening.depth_score}/3 | {r.deepening.valley_type:<12} | {'โœ“' if r.depth_transferable else 'โœ—':<10} | {r.curriculum_recommendation[:30]}") + +# Final analysis +print("\n" + "=" * 70) +print("๐ŸŒ™ CURRICULUM IMPLICATIONS") +print("=" * 70) + +# Group by recommendation +high_depth = [r for r in results if r.depth_accessible and r.depth_transferable] +needs_work = [r for r in results if not r.depth_accessible] +needs_bridge = [r for r in results if r.depth_accessible and not r.depth_transferable] + +if high_depth: + print(f"\nโœ… READY FOR MULTILINGUAL CURRICULUM:") + for r in high_depth: + print(f" - {r.concept}: {r.curriculum_recommendation}") + +if needs_bridge: + print(f"\nโš ๏ธ NEEDS BRIDGING (depth accessible but not transferable):") + for r in needs_bridge: + print(f" - {r.concept}: depth={r.deepening.depth_score} but transfer failed") + +if needs_work: + print(f"\nโŒ NEEDS GROUNDING FIRST:") + for r in needs_work: + print(f" - {r.concept}: depth only {r.deepening.depth_score}/3") + +print("\n" + "=" * 70) +print("โœ… TRIANGULATION PROBE TEST COMPLETE") +print("=" * 70)