Add moses-sampler scaffold (copy + organize processors)

moses-sampler/README.md

@@ -0,0 +1,37 @@
+# moses-sampler
+
+A staging ground that gathers four MO§ES-lineage processors into one place so
+they can be sampled behind a single Gradio app. **Nothing is wired yet** — this
+pass only copies and organizes the source material.
+
+## Structure
+
+```
+moses-sampler/
+├── app.py              # empty Gradio scaffold (boots, does nothing yet)
+├── requirements.txt    # scaffold deps only (gradio)
+├── processors/
+│   ├── conservation/   # commitment conservation harness
+│   ├── sigarmy/        # signal_army + sigsystem
+│   ├── sigtoken/       # sig_token core
+│   └── governance/     # MO§ES governance plugin
+└── README.md
+```
+
+## Provenance
+
+Files were copied verbatim from their source repositories:
+
+| Processor      | Source repo               | Source path                                              |
+|----------------|---------------------------|----------------------------------------------------------|
+| `conservation` | `commitment-conservation` | `operational-harness/src/` (entire directory)            |
+| `sigarmy`      | `RNS`                      | `2_secondary/sig_army/main/signal_army/signal_army.py`   |
+| `sigarmy`      | `RNS`                      | `2_secondary/sig_army/main/sigsystem/sigsystem.py`       |
+| `sigtoken`     | `RNS`                      | `2_secondary/sig_army/main/sigtoken/` (core) + `sigtoken_v2/` |
+| `governance`   | `moses-governance`         | core plugin files (manifests, hooks, scripts, modes, rules, agents, commands, skills, references) |
+
+## Status
+
+- [x] Copy + organize sources
+- [ ] Wire processors into `app.py`
+- [ ] Pin per-processor dependencies in `requirements.txt`

moses-sampler/app.py

@@ -0,0 +1,26 @@
+"""
+moses-sampler — Gradio scaffold.
+
+Empty entry point. Nothing is wired to the processors yet; this is a
+placeholder UI so the app boots. The four processor packages live under
+`processors/` and will be connected here in a later pass.
+"""
+
+import gradio as gr
+
+
+def build_ui() -> gr.Blocks:
+    with gr.Blocks(title="MO§ES Sampler") as demo:
+        gr.Markdown("# MO§ES Sampler")
+        gr.Markdown(
+            "Scaffold only — processors are copied in but not yet wired.\n\n"
+            "- `processors/conservation` — commitment conservation harness\n"
+            "- `processors/sigarmy` — signal_army + sigsystem\n"
+            "- `processors/sigtoken` — sig_token core\n"
+            "- `processors/governance` — MO§ES governance plugin"
+        )
+    return demo
+
+
+if __name__ == "__main__":
+    build_ui().launch()

moses-sampler/processors/conservation/__init__.py

@@ -0,0 +1,6 @@
+"""
+Commitment Conservation Test Harness
+
+Research evaluation harness for testing commitment preservation
+under compression and recursion.
+"""

moses-sampler/processors/conservation/advanced_extractor.py

@@ -0,0 +1,89 @@
+# ...new file...
+import re
+import json
+import hashlib
+import dateparser
+import spacy
+
+nlp = spacy.load("en_core_web_sm")
+
+NUM_RE = re.compile(r'\$?\d{1,3}(?:[,\d]*)?(?:\.\d+)?')
+
+MODAL_LEX = {
+    "must": "OBLIGATION", "shall": "OBLIGATION", "required": "OBLIGATION",
+    "must not": "PROHIBITION", "shall not": "PROHIBITION", "cannot": "PROHIBITION",
+    "may": "PERMISSION", "is defined as": "DEFINITION", "means": "DEFINITION"
+}
+
+def normalize_text(s: str) -> str:
+    s = s.strip()
+    s = s.replace("—", "-").replace("–", "-")
+    s = " ".join(s.split())
+    return s
+
+def canonicalize_number(tok: str) -> str:
+    # convert simple money/number patterns to placeholders
+    if NUM_RE.search(tok):
+        return "#NUM"
+    dt = dateparser.parse(tok)
+    if dt:
+        return dt.date().isoformat()
+    return tok.lower()
+
+def sentence_candidates(text: str):
+    doc = nlp(normalize_text(text))
+    return [sent.text.strip() for sent in doc.sents]
+
+def cue_lookup(sent: str):
+    s = sent.lower()
+    for cue, mod in MODAL_LEX.items():
+        if cue in s:
+            return cue, mod
+    return None, None
+
+def build_tuple_from_sentence(sent: str):
+    cue, modality = cue_lookup(sent)
+    doc = nlp(sent)
+    subj = None
+    obj = None
+    verb = None
+    cond = None
+    # regex conditional capture
+    m = re.search(r'(.+?)\b(if|when|provided that|unless|in the event that)\b(.+)', sent, flags=re.I)
+    if m:
+        cond = m.group(3).strip()
+    # dependency heuristics
+    for token in doc:
+        if token.dep_ in ("nsubj", "nsubjpass") and subj is None:
+            subj = token.text
+        if token.dep_ in ("dobj", "pobj", "attr") and obj is None:
+            obj = token.text
+        if token.pos_ == "VERB" and verb is None:
+            verb = token.lemma_
+    subj = subj or "UNKNOWN"
+    verb = verb or ""
+    obj = obj or ""
+    # canonicalize object tokens
+    obj_canon = " ".join(canonicalize_number(t.text) for t in nlp(obj)) if obj else ""
+    cond_canon = cond.lower() if cond else ""
+    tup = {
+        "actor": subj.lower(),
+        "modality": modality or "UNMARKED",
+        "action": verb,
+        "object": obj_canon,
+        "condition": cond_canon
+    }
+    # canonical key deterministic JSON
+    key = json.dumps(tup, sort_keys=True, separators=(',', ':'))
+    key_hash = hashlib.sha256(key.encode("utf8")).hexdigest()[:12]
+    return tup, key, key_hash
+
+def extract_hard(text: str):
+    keys = []
+    for sent in sentence_candidates(text):
+        cue, _ = cue_lookup(sent)
+        if cue:
+            tup, key, h = build_tuple_from_sentence(sent)
+            keys.append(key)
+    # deterministic fallback: if none, emit empty set
+    return set(keys)

moses-sampler/processors/conservation/config.py

@@ -0,0 +1,29 @@
+# config.py
+
+# Configuration settings for the commitment test harness project
+
+class Config:
+    # Model paths
+    HUGGINGFACE_MODEL_PATH = "facebook/bart-large-cnn"  # Example model for summarization
+    SPACY_MODEL = "en_core_web_sm"  # spaCy model for extraction
+
+    # Extraction parameters
+    EXTRACTION_PARAMS = {
+        "min_length": 5,
+        "max_length": 100,
+        "do_sample": False,
+    }
+
+    # Plotting settings
+    PLOTTING_SETTINGS = {
+        "title": "Commitment Fidelity vs Compression Threshold",
+        "xlabel": "Compression Threshold",
+        "ylabel": "Fidelity",
+        "xlim": (0, 1),
+        "ylim": (0, 1),
+        "grid": True,
+    }
+
+# Test harness parameters
+SIGMA_GRID = [120, 80, 40, 20, 10, 5]
+RECURSION_DEPTH = 8

moses-sampler/processors/conservation/deterministic_pipeline.py

@@ -0,0 +1,70 @@
+# ...new file...
+import os
+from transformers import pipeline
+from .extraction import extract_hard
+from .metrics import fid_hard, delta_hard
+from .plotting import plot_fid, plot_delta
+from . import config
+
+# initialize deterministic pipelines (no sampling)
+SUMMARIZER = pipeline("summarization", model="facebook/bart-large-cnn", framework="pt", device=-1)
+# back-translation paraphrase via Marian (en->de and de->en)
+EN_DE = pipeline("translation", model="Helsinki-NLP/opus-mt-en-de", tokenizer="Helsinki-NLP/opus-mt-en-de", framework="pt")
+DE_EN = pipeline("translation", model="Helsinki-NLP/opus-mt-de-en", tokenizer="Helsinki-NLP/opus-mt-de-en", framework="pt")
+
+def transform_sieve(text, sigma):
+    # Summarization (compression)
+    summ = SUMMARIZER(text, max_length=sigma, min_length=max(5, sigma//4), do_sample=False)[0]['summary_text']
+    # Paraphrase via back-translation
+    de = EN_DE(summ, max_length=400, do_sample=False)[0]['translation_text']
+    para = DE_EN(de, max_length=400, do_sample=False)[0]['translation_text']
+    # Abstraction: simple extractive shortener (first sentence)
+    abstract = summ.split(".")[0].strip()
+    return [summ, para, abstract]
+
+def compression_sweep(signal_text):
+    base = extract_hard(signal_text)
+    sig_label = signal_text[:40].replace("\n"," ")
+    sigma_vals = []
+    fid_vals = []
+    for s in config.SIGMA_GRID:
+        outs = transform_sieve(signal_text, s)
+        # intersection across transforms per protocol
+        sets = [extract_hard(o) for o in outs]
+        if sets:
+            inter = set.intersection(*sets) if all(sets) else set()
+        else:
+            inter = set()
+        fid = fid_hard(base, inter)
+        sigma_vals.append(s)
+        fid_vals.append(fid)
+    plot_fid(sig_label, sigma_vals, fid_vals, outpath=f"fid_{hash(sig_label)}.png")
+    return sigma_vals, fid_vals
+
+def recursion_test(signal_text, depth=config.RECURSION_DEPTH, enforced=False):
+    base = extract_hard(signal_text)
+    cur = signal_text
+    deltas = []
+    for n in range(depth+1):
+        cur_keys = extract_hard(cur)
+        deltas.append(delta_hard(base, cur_keys))
+        if n==depth:
+            break
+        # next step
+        if enforced:
+            # simple enforcement: prepend canonicalized base keys as context marker
+            marker = "COMMITMENT_HASH:" + str(hash("".join(sorted(base))))
+            ctx = marker + " " + cur
+        else:
+            ctx = cur
+        # use summarizer as step transform to simulate T
+        next_s = SUMMARIZER(ctx, max_length=40, min_length=5, do_sample=False)[0]['summary_text']
+        cur = next_s
+    plot_delta(signal_text[:30], list(range(depth+1)), deltas, outpath=f"delta_{hash(signal_text[:30])}.png")
+    return deltas
+
+if __name__ == "__main__":
+    for s in config.SIGNS["sample_signals"]:
+        compression_sweep(s)
+        recursion_test(s, enforced=False)
+        recursion_test(s, enforced=True)

moses-sampler/processors/conservation/extraction.py

@@ -0,0 +1,48 @@
+from spacy import load
+import re
+
+def load_spacy_model(model_name='en_core_web_sm'):
+    nlp = load(model_name)
+    return nlp
+
+def normalize_text(text):
+    """Normalize text for comparison: lowercase, strip punctuation."""
+    return re.sub(r'[^\w\s]', '', text.lower().strip())
+
+def extract_hard_commitments(text, nlp=None):
+    """Extract commitments using expanded modal keyword detection."""
+    if nlp is None:
+        nlp = load_spacy_model()
+
+    doc = nlp(text)
+    commitments = set()
+
+    # Expanded modal keywords
+    hard_modals = {'must', 'shall', 'will', 'have', 'need', 'required', 'ought', 'cannot', 'should'}
+    soft_modals = {'might', 'could', 'may', 'perhaps', 'maybe', 'tend'}
+
+    # Extract by sentence-level modal presence
+    for sent in doc.sents:
+        sent_lower = sent.text.lower()
+        # Check for hard modals
+        if any(modal in sent_lower for modal in hard_modals):
+            commitments.add(sent.text.strip())
+        # Check for soft modals
+        elif any(modal in sent_lower for modal in soft_modals):
+            commitments.add(sent.text.strip())
+
+    return commitments
+
+def extract_from_texts(texts, model_name='en_core_web_sm'):
+    nlp = load_spacy_model(model_name)
+    all_commitments = {}
+
+    for text in texts:
+        commitments = extract_hard_commitments(text, nlp)
+        all_commitments[text] = commitments
+
+    return all_commitments
+
+def extract_hard(text: str, nlp=None) -> set:
+    """Shorthand for extract_hard_commitments."""
+    return extract_hard_commitments(text, nlp)

moses-sampler/processors/conservation/harness.py

@@ -0,0 +1,49 @@
+"""
+Research Evaluation Harness
+
+This code is provided for academic and research evaluation purposes only.
+It implements the experimental harness described in the accompanying paper.
+
+This code is not intended for production deployment.
+"""
+
+from transformers import pipeline
+import spacy
+from .metrics import jaccard_index
+import matplotlib.pyplot as plt
+
+def run_tests(signal, compression_thresholds):
+    summarizer = pipeline("summarization")
+    nlp = spacy.load("en_core_web_sm")
+
+    original_commitments = extract_hard_commitments(signal, nlp)
+    fidelity_results = []
+
+    for threshold in compression_thresholds:
+        compressed_signal = compress_signal(signal, threshold, summarizer)
+        compressed_commitments = extract_hard_commitments(compressed_signal, nlp)
+        fidelity = jaccard_index(original_commitments, compressed_commitments)
+        fidelity_results.append(fidelity)
+
+    plot_results(compression_thresholds, fidelity_results)
+
+def extract_hard_commitments(signal, nlp):
+    doc = nlp(signal)
+    commitments = set()
+    for sent in doc.sents:
+        # Example extraction logic; customize as needed
+        commitments.add(sent.text)
+    return commitments
+
+def compress_signal(signal, threshold, summarizer):
+    # Example compression logic; customize as needed
+    summary = summarizer(signal, max_length=threshold, min_length=5, do_sample=False)
+    return summary[0]['summary_text']
+
+def plot_results(thresholds, fidelity):
+    plt.plot(thresholds, fidelity, marker='o')
+    plt.title('Fidelity of Hard Commitments vs Compression Threshold')
+    plt.xlabel('Compression Threshold')
+    plt.ylabel('Jaccard Fidelity')
+    plt.grid()
+    plt.close()  # Use close() instead of show() to avoid blocking in tests