Memory & Search

def _get_similar_by_embedding(query, candidates, model='nomic-embed-text',
                              provider='ollama', top_k=20):
    """Pre-filter candidates by embedding cosine similarity.

    Returns top-K candidate strings most similar to query.
    Falls back to returning all candidates if embedding fails.
    """
    if not candidates or len(candidates) <= top_k:
        return list(candidates)

    try:
        from npcpy.gen.embeddings import get_embeddings

        query_emb = np.array(get_embeddings([query], model, provider)[0])
        cand_embs = get_embeddings(list(candidates), model, provider)

        similarities = []
        for i, emb in enumerate(cand_embs):
            emb_arr = np.array(emb)
            norm_product = np.linalg.norm(query_emb) * np.linalg.norm(emb_arr)
            if norm_product > 0:
                sim = float(np.dot(query_emb, emb_arr) / norm_product)
            else:
                sim = 0.0
            similarities.append((sim, i))

        similarities.sort(key=lambda x: -x[0])
        return [candidates[idx] for _, idx in similarities[:top_k]]
    except Exception as e:
        logger.warning(f"Embedding pre-filter failed, using all candidates: {e}")
        return list(candidates)

def abstract(groups, 
             model, 
             provider, 
             npc=None,
             context: str = None, 
             **kwargs):
    """
    Create more abstract terms from groups.
    """
    sample_groups = random.sample(groups, min(len(groups), max(3, len(groups) // 2)))

    groups_text_for_prompt = "\n".join([f'- "{g["name"]}"' for g in sample_groups])

    prompt = f"""
        Create more abstract categories from this list of groups.

        Groups:
        {groups_text_for_prompt}

        You will create higher-level concepts that interrelate between the given groups. 

        Create abstract categories that encompass multiple related facts, but do not unnecessarily combine facts with conjunctions. For example, do not try to combine "characters", "settings", and "physical reactions" into a
        compound group like "Characters, Setting, and Physical Reactions". This kind of grouping is not productive and only obfuscates true abstractions. 
        For example, a group that might encompass the three aforermentioned names might be "Literary Themes" or "Video Editing Functionis", depending on the context.
        Your aim is to abstract, not to just arbitrarily generate associations. 

        Group names should never be more than two words. They should not contain gerunds. They should never contain conjunctions like "AND" or "OR".
        Generate no more than 5 new concepts and no fewer than 2. 

        Respond with JSON:
        """ + '{"groups": [{"name": "abstract category name"}]}'

    response = get_llm_response(prompt,
                                model=model,
                                provider=provider,
                                format="json",
                                npc=npc,
                                context=context,
                                **kwargs)

    return response["response"].get("groups", [])

def consolidate_facts_llm(new_fact, 
                          existing_facts, 
                          model, 
                          provider, 
                          npc=None,
                          context: str =None,
                          **kwargs):
    """
    Uses an LLM to decide if a new fact is novel or redundant.
    """
    prompt = f"""
        Analyze the "New Fact" in the context of the "Existing Facts" list.
        Your task is to determine if the new fact provides genuinely new information or if it is essentially a repeat or minor rephrasing of information already present.

        New Fact:
        "{new_fact['statement']}"

        Existing Facts:
        {json.dumps([f['statement'] for f in existing_facts], indent=2)}

        Possible decisions:
        - 'novel': The fact introduces new, distinct information not covered by the existing facts.
        - 'redundant': The fact repeats information already present in the existing facts.

        Respond with a JSON object:
        """ + '{"decision": "novel or redundant", "reason": "A brief explanation for your decision."}'
    response = get_llm_response(prompt,
                                model=model, 
                                provider=provider, 
                                format="json", 
                                npc=npc,
                                context=context,
                                **kwargs)
    return response['response']

def find_similar_facts_chroma(
    collection,
    query: str,
    query_embedding: List[float],
    n_results: int = 5,
    metadata_filter: Optional[Dict] = None,
) -> List[Dict]:
    """Find facts similar to the query using pre-generated embedding."""
    try:
        results = collection.query(
            query_embeddings=[query_embedding],
            n_results=n_results,
            where=metadata_filter,
        )

        formatted_results = []
        for i, doc in enumerate(results["documents"][0]):
            formatted_results.append(
                {
                    "fact": doc,
                    "metadata": results["metadatas"][0][i],
                    "id": results["ids"][0][i],
                    "distance": (
                        results["distances"][0][i] if "distances" in results else None
                    ),
                }
            )

        return formatted_results
    except Exception as e:
        logger.warning(f"Error searching in Chroma: {e}")
        return []

def generate_groups(facts, 
                    model=None,
                    provider=None,
                    npc=None,
                    context: str =None, 
                    **kwargs):
    """Generate conceptual groups for facts"""

    facts_text = "\n".join([f"- {fact['statement']}" for fact in facts])

    prompt = f"""
    Generate conceptual groups for this group off facts:

    {facts_text}

    Create categories that encompass multiple related facts, but do not unnecessarily combine facts with conjunctions. 

    Your aim is to generalize commonly occurring ideas into groups, not to just arbitrarily generate associations. 
    Focus on the key commonly occurring items and expresions.     

    Group names should never be more than two words. They should not contain gerunds. They should never contain conjunctions like "AND" or "OR".
    Respond with JSON:
    """ + '{"groups": [{"name": "group name"}]}'

    response = get_llm_response(prompt,
                                model=model,
                                provider=provider,
                                format="json",
                                context=context,
                                npc=npc,
                                **kwargs)

    return response["response"].get("groups", [])

def get_facts(content_text, 
              model= None,
              provider = None,
              npc=None,
              context : str=None, 
              attempt_number=1,
              n_attempts=3,

              **kwargs):
    """Extract facts from content text"""

    prompt = f"""
    Extract facts from this text. A fact is a specific statement that can be sourced from the text.

    Example: if text says "the moon is the earth's only currently known satellite", extract:
    - "The moon is a satellite of earth" 
    - "The moon is the only current satellite of earth"
    - "There may have been other satellites of earth" (inferred from "only currently known")

        A fact is a piece of information that makes a statement about the world.
        A fact is typically a sentence that is true or false.
        Facts may be simple or complex. They can also be conflicting with each other, usually
        because there is some hidden context that is not mentioned in the text.
        In any case, it is simply your job to extract a list of facts that could pertain to
        an individual's personality.

        For example, if a message says:
            "since I am a doctor I am often trying to think up new ways to help people.
            Can you help me set up a new kind of software to help with that?"
        You might extract the following facts:
            - The individual is a doctor
            - They are helpful

        Another example:
            "I am a software engineer who loves to play video games. I am also a huge fan of the
            Star Wars franchise and I am a member of the 501st Legion."
        You might extract the following facts:
            - The individual is a software engineer
            - The individual loves to play video games
            - The individual is a huge fan of the Star Wars franchise
            - The individual is a member of the 501st Legion

        Another example:
            "The quantum tunneling effect allows particles to pass through barriers
            that classical physics says they shouldn't be able to cross. This has
            huge implications for semiconductor design."
        You might extract these facts:
            - Quantum tunneling enables particles to pass through barriers that are
              impassable according to classical physics
            - The behavior of quantum tunneling has significant implications for
              how semiconductors must be designed

        Another example:
            "People used to think the Earth was flat. Now we know it's spherical,
            though technically it's an oblate spheroid due to its rotation."
        You might extract these facts:
            - People historically believed the Earth was flat
            - It is now known that the Earth is an oblate spheroid
            - The Earth's oblate spheroid shape is caused by its rotation

        Another example:
            "My research on black holes suggests they emit radiation, but my professor
            says this conflicts with Einstein's work. After reading more papers, I
            learned this is actually Hawking radiation and doesn't conflict at all."
        You might extract the following facts:
            - Black holes emit radiation
            - The professor believes this radiation conflicts with Einstein's work
            - The radiation from black holes is called Hawking radiation
            - Hawking radiation does not conflict with Einstein's work

        Another example:
            "During the pandemic, many developers switched to remote work. I found
            that I'm actually more productive at home, though my company initially
            thought productivity would drop. Now they're keeping remote work permanent."
        You might extract the following facts:
            - The pandemic caused many developers to switch to remote work
            - The individual discovered higher productivity when working from home
            - The company predicted productivity would decrease with remote work
            - The company decided to make remote work a permanent option

        Thus, it is your mission to reliably extract lists of facts.

    Here is the text:
    Text: "{content_text}"

    Facts should never be more than one or two sentences, and they should not be overly complex or literal. They must be explicitly
    derived or inferred from the source text. Do not simply repeat the source text verbatim when stating the fact. 

    No two facts should share substantially similar claims. They should be conceptually distinct and pertain to distinct ideas, avoiding lengthy convoluted or compound facts .
    Respond with JSON:
    """ + '{"facts": [{"statement": "fact statement that builds on input text to state a specific claim that can be falsified through reference to the source material", "source_text": "text snippets related to the source text", "type": "explicit or inferred"}]}'

    response = get_llm_response(prompt, 
                                model=model,
                                provider=provider, 
                                npc=npc,
                                format="json", 
                                context=context,
                                **kwargs)

    if len(response.get("response", {}).get("facts", [])) == 0 and attempt_number < n_attempts:
        print(f"  Attempt {attempt_number} to extract facts yielded no results. Retrying...")
        return get_facts(content_text, 
                         model=model, 
                         provider=provider, 
                         npc=npc,
                         context=context,
                         attempt_number=attempt_number+1,
                         n_attempts=n_attempts,
                         **kwargs)

    return response["response"].get("facts", [])

def get_llm_response(
    prompt: str,
    model: str = None,
    provider: str = None,
    images: List[str] = None,
    npc: Any = None,
    team: Any = None,
    messages: List[Dict[str, str]] = None,
    api_url: str = None,
    api_key: str = None,
    context=None,
    stream: bool = False,
    attachments: List[str] = None,
    include_usage: bool = False,
    n_samples: int = 1,
    matrix: Optional[Dict[str, List[Any]]] = None,
    **kwargs,
):
    """Generate a response using the specified provider and model."""
    logger.debug(
        f"[get_llm_response] {len(messages) if messages else 0} messages, "
        f"prompt_len={len(prompt) if prompt else 0}, "
        f"context={'yes' if context else 'no'}"
    )

    base_model, base_provider, base_api_url, base_api_key = resolve_model_provider(
        npc=npc, team=team, model=model, provider=provider,
        api_url=api_url, api_key=api_key, images=images, attachments=attachments,
    )
    if api_key is None:
        api_key = base_api_key

    use_matrix = matrix is not None and len(matrix) > 0
    multi_sample = n_samples and n_samples > 1

    if not use_matrix and not multi_sample:
        # Simple single-call path
        run_model, run_provider, run_api_url, run_api_key = resolve_model_provider(
            npc=npc, team=team, model=base_model, provider=base_provider,
            api_url=api_url, api_key=api_key, images=images, attachments=attachments,
        )
        tool_capable = bool(kwargs.get("tools"))
        system_message = get_system_message(npc, team, tool_capable=tool_capable) if npc is not None else "You are a helpful assistant."

        # Build messages
        run_messages = copy.deepcopy(messages) if messages else []
        if not run_messages:
            run_messages = [{"role": "system", "content": system_message}]

        # Build full text from prompt and/or context
        full_text = ""
        if prompt and context:
            full_text = f"{prompt}\n\n\nUser Provided Context: {context}"
        elif prompt:
            full_text = prompt
        elif context:
            full_text = f"User Provided Context: {context}"

        if full_text:
            if run_messages[-1]["role"] == "user" and isinstance(run_messages[-1]["content"], str):
                run_messages[-1]["content"] += "\n" + full_text
            else:
                run_messages.append({"role": "user", "content": full_text})

        if not run_model:
            raise ValueError("No model specified. Please set a model in your NPC configuration or team settings.")
        return get_litellm_response(
            full_text or None,
            messages=run_messages,
            model=run_model,
            provider=run_provider,
            api_url=run_api_url,
            api_key=api_key,
            images=images,
            attachments=attachments,
            stream=stream,
            include_usage=include_usage,
            **kwargs,
        )

    # Matrix / multi-sample path
    combos = []
    if use_matrix:
        keys = list(matrix.keys())
        values = []
        for key in keys:
            val = matrix[key]
            if isinstance(val, (list, tuple, set)):
                values.append(list(val))
            else:
                values.append([val])
        for combo_values in itertools.product(*values):
            combos.append(dict(zip(keys, combo_values)))
    else:
        combos.append({})

    runs = []
    for combo in combos:
        run_npc = combo.get("npc", npc)
        run_team = combo.get("team", team)
        run_context = combo.get("context", context)
        extra_kwargs = dict(kwargs)
        for k, v in combo.items():
            if k not in {"model", "provider", "npc", "context", "team"}:
                extra_kwargs[k] = v

        run_model, run_provider, run_api_url, run_api_key = resolve_model_provider(
            npc=run_npc, team=run_team,
            model=combo.get("model", base_model),
            provider=combo.get("provider", base_provider),
            api_url=api_url, api_key=api_key, images=images, attachments=attachments,
        )

        tool_capable = bool(extra_kwargs.get("tools"))
        system_message = get_system_message(run_npc, run_team, tool_capable=tool_capable) if run_npc is not None else "You are a helpful assistant."

        run_messages = copy.deepcopy(messages) if messages else []
        if not run_messages:
            run_messages = [{"role": "system", "content": system_message}]

        full_text = ""
        if prompt and run_context:
            full_text = f"{prompt}\n\n\nUser Provided Context: {run_context}"
        elif prompt:
            full_text = prompt
        elif run_context:
            full_text = f"User Provided Context: {run_context}"

        if full_text:
            if run_messages[-1]["role"] == "user" and isinstance(run_messages[-1]["content"], str):
                run_messages[-1]["content"] += "\n" + full_text
            else:
                run_messages.append({"role": "user", "content": full_text})

        for sample_idx in range(max(1, n_samples or 1)):
            resp = get_litellm_response(
                full_text or None,
                messages=run_messages,
                model=run_model,
                provider=run_provider,
                api_url=run_api_url,
                api_key=api_key,
                images=images,
                attachments=attachments,
                stream=stream,
                include_usage=include_usage,
                **extra_kwargs,
            )
            runs.append({
                "response": resp.get("response") if isinstance(resp, dict) else resp,
                "raw": resp,
                "combo": combo,
                "sample_index": sample_idx,
            })

    aggregated = {
        "response": runs[0]["response"] if runs else None,
        "runs": runs,
    }
    if runs and isinstance(runs[0]["raw"], dict) and "messages" in runs[0]["raw"]:
        aggregated["messages"] = runs[0]["raw"].get("messages")
    return aggregated

def get_related_concepts_multi(node_name: str, 
                               node_type: str, 
                               all_concept_names, 
                               model: str = None,
                               provider: str = None,
                               npc=None,
                               context : str = None, 
                               **kwargs):
    """Links any node (fact or concept) to ALL relevant concepts in the entire ontology."""
    json_format = 'Respond with JSON: {"related_concepts": ["Concept A", "Concept B", ...]}'
    prompt = f"""
    Which of the following concepts from the entire ontology relate to the given {node_type}?
    Select all that apply, from the most specific to the most abstract.

    {node_type.capitalize()}: "{node_name}"

    Available Concepts:
    {json.dumps(all_concept_names, indent=2)}

    {json_format}
    """
    response = get_llm_response(prompt, 
                                model=model, 
                                provider=provider, 
                                format="json", 
                                npc=npc,
                                context=context, 
                                **kwargs)
    return response["response"].get("related_concepts", [])

def get_related_facts_llm(new_fact_statement, 
                          existing_fact_statements, 
                          model = None, 
                          provider = None,
                          npc = None, 
                          attempt_number = 1,
                          n_attempts = 3,
                          context='', 
                          **kwargs):
    """Identifies which existing facts are causally or thematically related to a new fact."""
    prompt = f"""
    A new fact has been learned: "{new_fact_statement}"

    Which of the following existing facts are directly related to it (causally, sequentially, or thematically)?
    Select only the most direct and meaningful connections.

    Existing Facts:
    {json.dumps(existing_fact_statements, indent=2)}

    Respond with JSON:
    """ + '{"related_facts": ["statement of a related fact", ...]}'
    response = get_llm_response(prompt,
                                model=model, 
                                provider=provider, 
                                format="json", 
                                npc=npc,
                                context=context,
                                **kwargs)   
    if attempt_number <= n_attempts:
        if not response["response"].get("related_facts", []):
            print(f"  Attempt {attempt_number} to find related facts yielded no results. Retrying...")
            return get_related_facts_llm(new_fact_statement, 
                                           existing_fact_statements, 
                                           model=model, 
                                           provider=provider, 
                                           npc=npc,
                                           attempt_number=attempt_number+1,
                                           n_attempts=n_attempts,
                                           context=context,
                                           **kwargs)    

    return response["response"].get("related_facts", [])

def kg_add_concept(
   engine,
   concept_name: str,
   concept_description: str,
   npc=None,
   team=None,
   model=None,
   provider=None
):
   """Add a new concept to the knowledge graph"""
   directory_path = os.getcwd()
   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
   npc_name = npc.name if npc else 'default_npc'

   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)

   new_concept = {
       "name": concept_name,
       "description": concept_description,
       "generation": kg_data.get('generation', 0)
   }

   kg_data['concepts'].append(new_concept)
   save_kg_to_db(engine, kg_data, team_name, npc_name, directory_path)

   return f"Added concept: {concept_name}"

def kg_add_fact(
   engine,
   fact_text: str,
   npc=None,
   team=None,
   model=None,
   provider=None
):
   """Add a new fact to the knowledge graph"""
   directory_path = os.getcwd()
   team_name = getattr(team, 'name', 'default_team') if team else 'default_team'
   npc_name = npc.name if npc else 'default_npc'

   kg_data = load_kg_from_db(engine, team_name, npc_name, directory_path)

   new_fact = {
       "statement": fact_text,
       "source_text": fact_text,
       "type": "manual",
       "generation": kg_data.get('generation', 0),
       "origin": "manual_add"
   }

   kg_data['facts'].append(new_fact)
   save_kg_to_db(engine, kg_data, team_name, npc_name, directory_path)

   return f"Added fact: {fact_text}"

def kg_backfill_from_memories(
    engine,
    model: str = None,
    provider: str = None,
    npc=None,
    get_concepts: bool = True,
    link_concepts_facts: bool = False,
    link_concepts_concepts: bool = False,
    link_facts_facts: bool = False,
    dry_run: bool = False,
    context: str = ''
):
    """Backfill KG from approved memories that haven't been incorporated yet."""
    from sqlalchemy import text

    stats = {
        'scopes_processed': 0,
        'facts_before': 0,
        'facts_after': 0,
        'concepts_before': 0,
        'concepts_after': 0,
        'scopes': []
    }

    with engine.connect() as conn:
        stats['facts_before'] = conn.execute(text("SELECT COUNT(*) FROM kg_facts")).scalar() or 0
        stats['concepts_before'] = conn.execute(text("SELECT COUNT(*) FROM kg_concepts")).scalar() or 0

    with engine.connect() as conn:
        result = conn.execute(text("""
            SELECT id, npc, team, directory_path, initial_memory, final_memory
            FROM memory_lifecycle
            WHERE status IN ('human-approved', 'human-edited')
            ORDER BY npc, team, directory_path
        """))

        from collections import defaultdict
        memories_by_scope = defaultdict(list)
        for row in result:
            statement = row.final_memory or row.initial_memory
            scope = (row.npc or 'default', row.team or 'global_team', row.directory_path or os.getcwd())
            memories_by_scope[scope].append({
                'statement': statement,
                # Memory IS the fact — keep source_text populated so any existing consumer that
                # only reads source_text still gets the memory text.
                'source_text': statement,
                'memory_id': row.id,
                'type': 'explicit',
                'generation': 0
            })

    if dry_run:
        for scope, facts in memories_by_scope.items():
            stats['scopes'].append({
                'scope': scope,
                'memory_count': len(facts)
            })
        stats['scopes_processed'] = len(memories_by_scope)
        return stats

    for (npc_name, team_name, directory_path), facts in memories_by_scope.items():
        existing_kg = load_kg_from_db(engine, team_name, npc_name, directory_path)

        existing_statements = {f['statement'] for f in existing_kg.get('facts', [])}
        new_facts = [f for f in facts if f['statement'] not in existing_statements]

        if not new_facts:
            continue

        try:
            evolved_kg, _ = kg_evolve_incremental(
                existing_kg=existing_kg,
                new_facts=new_facts,
                model=model or (npc.model if npc else None),
                provider=provider or (npc.provider if npc else None),
                npc=npc,
                context=context,
                get_concepts=get_concepts,
                link_concepts_facts=link_concepts_facts,
                link_concepts_concepts=link_concepts_concepts,
                link_facts_facts=link_facts_facts
            )
            save_kg_to_db(engine, evolved_kg, team_name, npc_name, directory_path)

            stats['scopes'].append({
                'scope': (npc_name, team_name, directory_path),
                'facts_added': len(new_facts),
                'concepts_added': len(evolved_kg.get('concepts', [])) - len(existing_kg.get('concepts', []))
            })
            stats['scopes_processed'] += 1

        except Exception as e:
            logger.warning(f"Error processing scope {npc_name}/{team_name}: {e}")

    with engine.connect() as conn:
        stats['facts_after'] = conn.execute(text("SELECT COUNT(*) FROM kg_facts")).scalar() or 0
        stats['concepts_after'] = conn.execute(text("SELECT COUNT(*) FROM kg_concepts")).scalar() or 0

    return stats

def kg_dream_process(existing_kg,
                     model=None,
                     provider=None,
                     npc=None,
                     context='',
                     num_seeds=3):
    current_gen = existing_kg.get('generation', 0)
    next_gen = current_gen + 1
    logger.info(f"DREAMING (Creative Synthesis): Gen {current_gen} -> Gen {next_gen}")
    concepts = existing_kg.get('concepts', [])
    if len(concepts) < num_seeds:
        logger.info(f"Not enough concepts ({len(concepts)}) for dream. Skipping.")
        return existing_kg, {}
    seed_concepts = random.sample(concepts, k=num_seeds)
    seed_names = [c['name'] for c in seed_concepts]
    logger.info(f"Dream seeded with: {seed_names}")
    prompt = f"""
    Write a short, speculative paragraph (a 'dream') that plausibly connects the concepts of {json.dumps(seed_names)}.
    Invent a brief narrative or a hypothetical situation.
    Respond with JSON: {{"dream_text": "A short paragraph..."}}
    """
    response = get_llm_response(prompt,
                                model=model,
                                provider=provider, npc=npc,
                                format="json", context=context)
    dream_text = response['response'].get('dream_text')
    if not dream_text:
        logger.info("Failed to generate a dream narrative. Skipping.")
        return existing_kg, {}
    logger.info(f"Generated Dream: '{dream_text[:150]}...'")

    dream_kg, _ = kg_evolve_incremental(existing_kg, new_content_text=dream_text,
                                         model=model, provider=provider, npc=npc, context=context)

    original_fact_stmts = {f['statement'] for f in existing_kg['facts']}
    for fact in dream_kg['facts']:
        if fact['statement'] not in original_fact_stmts:
            fact['origin'] = 'dream'
    original_concept_names = {c['name'] for c in existing_kg['concepts']}
    for concept in dream_kg['concepts']:
        if concept['name'] not in original_concept_names:
            concept['origin'] = 'dream'
    logger.info("Dream analysis complete. New knowledge integrated.")
    return dream_kg, {}

def kg_embedding_search(
    engine,
    query: str,
    npc=None,
    team=None,
    embedding_model: str = None,
    embedding_provider: str = None,
    similarity_threshold: float = 0.6,
    max_results: int = 20,
    include_concepts: bool = True,
    search_all_scopes: bool = True,
):
    """Semantic search using embeddings via brute-force cosine similarity."""
    from sqlalchemy import text

    try:
        from npcpy.gen.embeddings import get_embeddings
    except ImportError:
        logger.warning("Embeddings not available, falling back to keyword search")
        facts = kg_search_facts(engine, query, npc=npc, team=team,
                               search_all_scopes=search_all_scopes)
        return [{'content': f, 'type': 'fact', 'score': 0.5} for f in facts[:max_results]]

    model = embedding_model or 'nomic-embed-text'
    provider = embedding_provider or 'ollama'

    team_name = getattr(team, 'name', None) if team else None
    npc_name = getattr(npc, 'name', None) if npc else None

    results = []

    query_embedding = np.array(get_embeddings([query], model, provider)[0])

    with engine.connect() as conn:
        if search_all_scopes:
            fact_rows = conn.execute(text(
                "SELECT DISTINCT statement FROM kg_facts"
            )).fetchall()
        else:
            t_name = team_name or 'global_team'
            n_name = npc_name or 'default_npc'
            fact_rows = conn.execute(text("""
                SELECT statement FROM kg_facts
                WHERE team_name = :team AND npc_name = :npc
            """), {"team": t_name, "npc": n_name}).fetchall()

        if fact_rows:
            statements = [r.statement for r in fact_rows]
            embeddings = get_embeddings(statements, model, provider)

            for i, stmt in enumerate(statements):
                emb = np.array(embeddings[i])
                norm_p = np.linalg.norm(query_embedding) * np.linalg.norm(emb)
                if norm_p > 0:
                    sim = float(np.dot(query_embedding, emb) / norm_p)
                    if sim >= similarity_threshold:
                        results.append({'content': stmt, 'type': 'fact', 'score': sim})

        if include_concepts:
            if search_all_scopes:
                concept_rows = conn.execute(text(
                    "SELECT DISTINCT name FROM kg_concepts"
                )).fetchall()
            else:
                concept_rows = conn.execute(text("""
                    SELECT name FROM kg_concepts
                    WHERE team_name = :team AND npc_name = :npc
                """), {"team": t_name, "npc": n_name}).fetchall()

            if concept_rows:
                names = [r.name for r in concept_rows]
                embeddings = get_embeddings(names, model, provider)

                for i, name in enumerate(names):
                    emb = np.array(embeddings[i])
                    norm_p = np.linalg.norm(query_embedding) * np.linalg.norm(emb)
                    if norm_p > 0:
                        sim = float(np.dot(query_embedding, emb) / norm_p)
                        if sim >= similarity_threshold:
                            results.append({'content': name, 'type': 'concept', 'score': sim})

    results.sort(key=lambda x: -x['score'])
    return results[:max_results]

def kg_evolve_incremental(existing_kg,
                          new_content_text=None,
                          new_facts=None,
                          model=None,
                          provider=None,
                          npc=None,
                          context='',
                          get_concepts=False,
                          link_concepts_facts=False,
                          link_concepts_concepts=False,
                          link_facts_facts=False,
                          embedding_model=None,
                          embedding_provider=None):

    current_gen = existing_kg.get('generation', 0)
    next_gen = current_gen + 1

    newly_added_concepts = []
    concept_links = list(existing_kg.get('concept_links', []))
    fact_to_concept_links = defaultdict(list,
                                        existing_kg.get('fact_to_concept_links', {}))
    fact_to_fact_links = list(existing_kg.get('fact_to_fact_links', []))

    existing_facts = existing_kg.get('facts', [])
    existing_concepts = existing_kg.get('concepts', [])
    existing_concept_names = {c['name'] for c in existing_concepts}
    existing_fact_statements = [f['statement'] for f in existing_facts]
    all_concept_names = list(existing_concept_names)

    all_new_facts = []

    if new_facts:
        all_new_facts = new_facts
        logger.info(f'Using pre-approved facts: {len(all_new_facts)}')
    elif new_content_text:
        logger.info('Extracting facts from content...')
        if len(new_content_text) > 10000:
            for n in range(len(new_content_text) // 10000):
                content_to_sample = new_content_text[n * 10000:(n + 1) * 10000]
                facts = get_facts(content_to_sample,
                                model=model,
                                provider=provider,
                                npc=npc,
                                context=context)
                all_new_facts.extend(facts)
        else:
            all_new_facts = get_facts(new_content_text,
                                model=model,
                                provider=provider,
                                npc=npc,
                                context=context)
    else:
        logger.info("No new content or facts provided")
        return existing_kg, {}

    existing_stmts = {f['statement'] for f in existing_facts}
    for fact in all_new_facts:
        fact['generation'] = next_gen

    final_facts = existing_facts + [f for f in all_new_facts if f['statement'] not in existing_stmts]

    if get_concepts:
        logger.info('Generating groups...')
        candidate_concepts = generate_groups(all_new_facts,
                                            model=model,
                                            provider=provider,
                                            npc=npc,
                                            context=context)
        for cand_concept in candidate_concepts:
            cand_name = cand_concept['name']
            if cand_name in existing_concept_names:
                continue
            cand_concept['generation'] = next_gen
            newly_added_concepts.append(cand_concept)
            if link_concepts_concepts:
                related_concepts = get_related_concepts_multi(cand_name,
                                                            "concept",
                                                            all_concept_names,
                                                            model,
                                                            provider,
                                                            npc,
                                                            context)
                for related_name in related_concepts:
                    if related_name != cand_name:
                        concept_links.append((cand_name, related_name))
            all_concept_names.append(cand_name)

        final_concepts = existing_concepts + newly_added_concepts

        if link_concepts_facts:
            for fact in all_new_facts:
                fact_to_concept_links[fact['statement']] = get_related_concepts_multi(
                    fact['statement'], "fact", all_concept_names,
                    model=model, provider=provider, npc=npc, context=context)
    else:
        final_concepts = existing_concepts

    if link_facts_facts and existing_fact_statements:
        e_model = embedding_model or 'nomic-embed-text'
        e_provider = embedding_provider or 'ollama'

        for new_fact in all_new_facts:
            # Pre-filter existing facts by embedding similarity
            candidates = _get_similar_by_embedding(
                new_fact['statement'], existing_fact_statements,
                e_model, e_provider, top_k=20)
            if candidates:
                related_fact_stmts = get_related_facts_llm(new_fact['statement'],
                                                           candidates,
                                                           model=model,
                                                           provider=provider,
                                                           npc=npc,
                                                           context=context)
                for related_stmt in related_fact_stmts:
                    fact_to_fact_links.append((new_fact['statement'], related_stmt))

    final_kg = {
        "generation": next_gen,
        "facts": final_facts,
        "concepts": final_concepts,
        "concept_links": concept_links,
        "fact_to_concept_links": dict(fact_to_concept_links),
        "fact_to_fact_links": fact_to_fact_links
    }
    return final_kg, {}