!pip install openai
copy

# Setup: store your token securely and create a DeepInfra OpenAI-compatible client
import os, getpass
from openai import OpenAI


os.environ["DEEPINFRA_API_TOKEN"] = getpass.getpass("Paste your DeepInfra API token: ")
client = OpenAI(
   api_key=os.environ["DEEPINFRA_API_TOKEN"],
   base_url="https://api.deepinfra.com/v1/openai",
copy

def ask_capped(messages, *, model="moonshotai/Kimi-K2-Instruct-0905", max_tokens=256):
   return client.chat.completions.create(
       model=model,
	messages=messages,
	max_tokens=max_tokens,                     # hard cap on output cost
	stop=["<END>", "\n\n\n"],                  # early stopping guard
	response_format={"type": "json_object"}    # terse, structured replies
   )
copy

msgs = [{"role":"user","content":"Give three bullet tips to cut LLM spend."}]
resp = ask_capped(msgs, max_tokens=120)
print(resp.choices[0].message.content)
copy

MODEL = "moonshotai/Kimi-K2-Instruct-0905"
# Make the shared prefix very long and ensure it's the very first tokens
SYSTEM_BLOCK = (
   "You are a concise JSON-only assistant. "
   "Respond with a compact JSON object. "
   "If additional detail exists, include \"expandable\": true. \n"
)
POLICY_LINE = (
   "Rule: Do not add commentary. Prefer short keys. "
   "Use stable field order. Avoid repeating unchanged values.\n"
)
SYSTEM_BLOCK = SYSTEM_BLOCK + (POLICY_LINE * 400)
# Put all reusable context into the *system* message (earliest position).
CONTEXT = (
   "Policy A: Users may request summaries; keep responses under 80 tokens by default.\n\n"
   "Policy B: For lists, cap items at 5 unless user asks to expand.\n\n"
   "Policy C: Use ISO dates; currencies in USD.\n"
)
def make_messages(user_question: str):
   return [
       {"role": "system", "content": SYSTEM_BLOCK + "\n" + CONTEXT},
       {"role": "user",   "content": user_question}
   ]
def call_once(q):
   r = client.chat.completions.create(
       model=MODEL,
       messages=make_messages(q),
       temperature=0,
       max_tokens=200,
   )
   u = r.usage
   pt  = getattr(u, "prompt_tokens", 0)
   ct  = getattr(u, "completion_tokens", 0)
   est = getattr(u, "estimated_cost", None)
   ptd = getattr(u, "prompt_tokens_details", None)
   cached = 0
   if ptd is not None:
       cached = getattr(ptd, "cached_tokens", 0) or (ptd.get("cached_tokens") if isinstance(ptd, dict) else 0)
   return {"pt": pt, "ct": ct, "cached": cached, "cost": float(est) if est is not None else None}


cold = call_once("Summarize the policies in 3 bullets.")
warm = call_once("List 3 risks if these rules are ignored.")  # different tail; identical prefix


print("Cold:", cold)  # expect low cache
print("Warm:", warm)  # expect very high cache
if cold["cost"] and warm["cost"]:
   print(f"Savings: {100*(1 - warm['cost']/cold['cost']):.1f}%")
copy

import uuid, os


# Unique per script run, identical across both calls in the run
RUN_SEED = os.getenv("RAG_RUN_SEED") or uuid.uuid4().hex


# Prepend the seed so this run's prefix doesn't reuse an older server cache
SYSTEM_BLOCK = f"RUN_SEED:{RUN_SEED}\n" + SYSTEM_BLOCK


import re, textwrap
from difflib import SequenceMatcher


# ──────────────────────────────────────────────────────────────────────────────
# RAG CHUNK PREP — de-duplicate near-identicals, cap K, prefer short spans
# ──────────────────────────────────────────────────────────────────────────────
SENTENCE_RE = re.compile(r"(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?|!)\s+")


def split_sentences(text: str):
   parts = SENTENCE_RE.split(text.strip())
   return [p.strip() for p in parts if p.strip()]


def _norm(s: str) -> str:
   return re.sub(r"\s+", " ", s).strip().lower()


def _near_dup(a: str, b: str, thresh: float = 0.92) -> bool:
   return SequenceMatcher(None, _norm(a), _norm(b)).ratio() >= thresh


def dedupe_cap_shorten(chunks, k=6, sentence_span=1, sim_threshold=0.92):
   """
   - Remove near-duplicates
   - Prefer short, sentence-level spans
   - Keep original order and cap to K
   """
   keep, seen = [], []
   for c in chunks:
       sents = split_sentences(c) or [c.strip()]
       short = " ".join(sents[:sentence_span])
       if any(_near_dup(short, prev, sim_threshold) for prev in seen):
           continue
       seen.append(short)
       keep.append(short)
       if len(keep) == k:
           break
   return keep


# ──────────────────────────────────────────────────────────────────────────────
# Message building with stable (cacheable) prefix + dynamic RAG block
# ──────────────────────────────────────────────────────────────────────────────
def make_messages(user_question: str, rag_chunks=None):
   rag_chunks = rag_chunks or []
   system_content = SYSTEM_BLOCK + "\n" + CONTEXT
   if rag_chunks:
       system_content += "\nRAG context:\n" + "\n".join(f"- {c}" for c in rag_chunks)
   return [
       {"role": "system", "content": system_content},
       {"role": "user",   "content": user_question}
   ]


def call_once(q, rag_chunks=None):
   r = client.chat.completions.create(
       model=MODEL,
       messages=make_messages(q, rag_chunks),
       temperature=0,
       max_tokens=200,
   )
   u = r.usage
   pt  = getattr(u, "prompt_tokens", 0)
   ct  = getattr(u, "completion_tokens", 0)
   est = getattr(u, "estimated_cost", None)
   ptd = getattr(u, "prompt_tokens_details", None)
   cached = 0
   if ptd is not None:
       # Works for both object- and dict-like payloads
       cached = getattr(ptd, "cached_tokens", 0) or (
           ptd.get("cached_tokens") if isinstance(ptd, dict) else 0
       )
   return {
       "pt": pt,
       "ct": ct,
       "cached": cached,
       "cost": float(est) if est is not None else None,
       "content": r.choices[0].message.content,
   }


# ──────────────────────────────────────────────────────────────────────────────
# DEMO — RAG cleanup + cold/warm cache comparison with provider-estimated cost
# ──────────────────────────────────────────────────────────────────────────────
if __name__ == "__main__":
   # Raw retrieval (has a duplicate and some wordiness)
   raw_chunks = [
       "Pricing depends on input/output token counts.",
       "Pricing depends on input/output token counts.",  # duplicate
       "Cached input reduces spend when reused exactly. Long passages are pricey; prefer short spans.",
       "Shorter system prompts cut cost.",
       "De-duplicate near-identical passages to avoid paying for repeats.",
       "Cap how many chunks you include.",
   ]


   # Clean: dedupe → cap → sentence-level
   ctx = dedupe_cap_shorten(raw_chunks, k=3, sentence_span=1)


   # Cold call (builds cache)
   cold = call_once("Summarize the policies in 3 bullets.", rag_chunks=ctx)
   # Warm call (identical long prefix, different user tail)
   warm = call_once("List 3 risks if these rules are ignored.", rag_chunks=ctx)


   print("\n--- Responses ---")
   print("Cold:", textwrap.shorten(cold["content"], width=140))
   print("Warm:", textwrap.shorten(warm["content"], width=140))


   print("\n--- Token & Cost (provider estimated) ---")
   print(f"Cold → pt={cold['pt']}, ct={cold['ct']}, cached={cold['cached']}, est_cost={cold['cost']!r}")
   print(f"Warm → pt={warm['pt']}, ct={warm['ct']}, cached={warm['cached']}, est_cost={warm['cost']!r}")
   if cold["cost"] and warm["cost"]:
       print(f"Savings: {100*(1 - warm['cost']/cold['cost']):.1f}%")


   print("\n--- RAG Context Sent ---")
   for i, c in enumerate(ctx, 1):
       print(f"{i}. {c}")
copy

import textwrap


# ── 1) Rolling window by characters ───────────────────────────────────────────
def trim_history(messages, max_chars=3500):
   """
   Keep the most-recent messages whose *cumulative* content length ≤ max_chars.
   Preserves order and always starts from the newest going backwards.
   """
   total, keep = 0, []
   for m in reversed(messages):
       total += len(m.get("content", ""))
       if total <= max_chars:
           keep.append(m)
       else:
           break
   return list(reversed(keep))


# ── 2) Optional: summarize older messages outside the window ──────────────────
def summarize_older_turns(older_msgs, max_tokens=120):
   """
   Compact the overflow into a short memory note you can prepend next turn.
   Returns a plain-text summary string (or None if nothing to summarize).
   """
   if not older_msgs:
       return None


   # Build a lightweight transcript to summarize
   transcript = []
   for m in older_msgs:
       role = m.get("role", "user")
       content = m.get("content", "")
       transcript.append(f"{role.upper()}: {content}")
   transcript_text = "\n".join(transcript)


   sys_prompt = (
       "You are a concise meeting scribe.\n"
       "Summarize the transcript into 4-6 bullets capturing facts, decisions, action items, and numbers.\n"
       "Be neutral. Keep ≤120 words. No introductions or conclusions."
   )
   messages = [
       {"role": "system", "content": sys_prompt},
       {"role": "user", "content": transcript_text},
   ]
   r = client.chat.completions.create(
       model=MODEL, messages=messages, temperature=0, max_tokens=max_tokens
   )
   return r.choices[0].message.content.strip()


# ── 3) Helper: call the model and print tokens/costs ──────────────────────────
def ask(messages, max_tokens=200, temperature=0):
   r = client.chat.completions.create(
       model=MODEL,
       messages=messages,
       temperature=temperature,
       max_tokens=max_tokens,
   )
   u = r.usage
   pt  = getattr(u, "prompt_tokens", 0)
   ct  = getattr(u, "completion_tokens", 0)
   est = getattr(u, "estimated_cost", None)
   ptd = getattr(u, "prompt_tokens_details", None)
   cached = 0
   if ptd is not None:
       cached = getattr(ptd, "cached_tokens", 0) or (
           ptd.get("cached_tokens") if isinstance(ptd, dict) else 0
       )
   print(f"→ pt={pt}, ct={ct}, cached={cached}, est_cost={float(est) if est is not None else None}")
   return r


# ── 4) Demo: maintain history across turns with trimming + optional summary ───
INSTRUCTIONS = (
   "You are a helpful assistant. Be brief but clear. "
   "If asked for steps, use short numbered lists."
)


# Seed a long conversation to trigger trimming
history = [
   {"role": "system", "content": INSTRUCTIONS},
   {"role": "user", "content": "We’re planning a launch. Product: Atlas. Date: 2025-12-01. Goals: awareness, signups."},
   {"role": "assistant", "content": "Got it. What’s the target audience and key channels?"},
   {"role": "user", "content": "Audience: data engineers in EU. Channels: blog, webinar, LinkedIn. Budget 25k EUR."},
   {"role": "assistant", "content": "Noted. Do we have a feature list or pricing tiers yet?"},
   {"role": "user", "content": "Features: real-time sync, schema inference, alerting. Pricing TBD; freemium likely."},
]
# Add some filler to exceed char budget (simulate long chit-chat)
for i in range(8):
   history.append({"role": "assistant", "content": f"Checkpoint {i}: recapping progress and minor notes..."})
   history.append({"role": "user", "content": f"More detail on webinar plan {i}: speakers, agenda, length, CTA, and follow-up emails."})


# New user turn arrives
history.append({"role": "user", "content": "Draft a tight 5-bullet launch checklist for Atlas."})


# Split into (older overflow) + (kept tail)
MAX_CHARS = 3500
kept = trim_history(history, max_chars=MAX_CHARS)


# Determine what got dropped and summarize it (optional)
dropped_len = len(history) - len(kept)
older = history[: max(0, len(history) - len(kept))]
summary_note = summarize_older_turns(older) if dropped_len > 0 else None


# Build final messages for this turn:
final_messages = []
final_messages.append({"role": "system", "content": INSTRUCTIONS})
if summary_note:
   final_messages.append({"role": "system", "content": "Conversation summary (memory):\n" + summary_note})


# Append the kept tail *excluding* the original system (we just re-added it)
for m in kept:
   if m["role"] == "system":
       continue
   final_messages.append(m)


# Ask the model with the trimmed+summarized context
resp = ask(final_messages, max_tokens=180, temperature=0)
print(textwrap.shorten(resp.choices[0].message.content, width=200))


# ── 5) Next turn example: keep summary & window rolling ───────────────────────
# User follows up; we append and repeat the process.
history = kept  # pretend we persisted; start from trimmed tail
if summary_note:
   # store your summary somewhere persistent and reuse on next turn
   pass


history.append({"role": "assistant", "content": resp.choices[0].message.content})
history.append({"role": "user", "content": "Great. Turn that into a 1-week timeline with owners."})


# Re-trim window
kept2 = trim_history(history, max_chars=MAX_CHARS)
final2 = [{"role": "system", "content": INSTRUCTIONS}]
if summary_note:
   final2.append({"role": "system", "content": "Conversation summary (memory):\n" + summary_note})
for m in kept2:
   if m["role"] == "system":
       continue
   final2.append(m)


resp2 = ask(final2, max_tokens=200, temperature=0)
print(textwrap.shorten(resp2.choices[0].message.content, width=200))
copy

def ask(messages, max_tokens=200, temperature=0):
   r = client.chat.completions.create(
       model=MODEL,
       messages=messages,
       temperature=temperature,
       max_tokens=max_tokens,
   )
   u = r.usage
   pt  = getattr(u, "prompt_tokens", 0)
   ct  = getattr(u, "completion_tokens", 0)
   est = getattr(u, "estimated_cost", None)
   ptd = getattr(u, "prompt_tokens_details", None)
   cached = 0
   if ptd is not None:
       cached = getattr(ptd, "cached_tokens", 0) or (
           ptd.get("cached_tokens") if isinstance(ptd, dict) else 0
       )
   print(f"→ pt={pt}, ct={ct}, cached={cached}, est_cost={float(est) if est is not None else None}")
   return r.choices[0].message.content


# seed full history (no trimming)
INSTRUCTIONS = (
   "You are a helpful assistant. Be brief but clear. "
   "If asked for steps, use short numbered lists."
)


history = [
   {"role": "system", "content": SYSTEM_BLOCK + "\n" + CONTEXT + "\nConversation starts now."},
   {"role": "user", "content": "We’re planning a launch. Product: Atlas. Date: 2025-12-01. Goals: awareness, signups."},
   {"role": "assistant", "content": "Noted. Who’s the audience and channels?"},
   {"role": "user", "content": "Audience: data engineers in EU. Channels: blog, webinar, LinkedIn. Budget 25k EUR."},
   {"role": "assistant", "content": "Understood. Any features or pricing?"},
   {"role": "user", "content": "Features: real-time sync, schema inference, alerting. Pricing TBD; freemium likely."},
]


# turn 1 (no rolling window — we send the entire history)
history.append({"role": "user", "content": "Draft a tight 5-bullet launch checklist for Atlas."})
resp1 = ask(history, max_tokens=200, temperature=0)
print(textwrap.shorten(resp1, width=160))
history.append({"role": "assistant", "content": resp1})


# turn 2 (history keeps growing; caching should be high for the long stable prefix)
history.append({"role": "user", "content": "Great. Turn that into a 1-week timeline with owners."})
resp2 = ask(history, max_tokens=200, temperature=0)
print(textwrap.shorten(resp2, width=160))
history.append({"role": "assistant", "content": resp2})
copy

# pip install openai
import os, csv, time, pathlib


# --- Config ---
MODEL = "moonshotai/Kimi-K2-Instruct-0905"
LOG_PATH = pathlib.Path("deepinfra_usage.csv")


def ask_capped(messages, max_tokens=40, temperature=0):
   """
   Minimal wrapper that returns the full response object.
   """
   return client.chat.completions.create(
       model=MODEL,
       messages=messages,
       temperature=temperature,
       max_tokens=max_tokens,
   )


def log_usage(resp, model):
   """
   Log usage.prompt_tokens, usage.completion_tokens, and estimated_cost (if present)
   to a CSV file: deepinfra_usage.csv
   """
   u = resp.usage
   # Some providers return `estimated_cost`; if not, this will be None
   est = getattr(u, "estimated_cost", None)
   row = [time.time(), model, getattr(u, "prompt_tokens", 0), getattr(u, "completion_tokens", 0), est]
   new = not LOG_PATH.exists()
   with LOG_PATH.open("a", newline="") as f:
       w = csv.writer(f)
       if new:
           w.writerow(["ts","model","prompt_tokens","completion_tokens","estimated_cost_usd"])
       w.writerow(row)


# --- Example run ---
if __name__ == "__main__":
   # 1) Simple one-liner request
   msgs = [{"role":"user","content":"One sentence on token pricing."}]
   r = ask_capped(msgs, max_tokens=40)
   print(r.choices[0].message.content)


   # 2) Log usage
   log_usage(r, MODEL)


   # 3) Inspect the provider's usage payload (may vary by SDK/provider)
   # Not all SDK objects expose __dict__; fall back to a safe projection
   try:
       print(r.usage.__dict__)    # works for many providers
   except Exception:
       u = r.usage
       print({
           "prompt_tokens": getattr(u, "prompt_tokens", None),
           "completion_tokens": getattr(u, "completion_tokens", None),
           "total_tokens": getattr(u, "total_tokens", None),
           "estimated_cost": getattr(u, "estimated_cost", None),
           "prompt_tokens_details": getattr(u, "prompt_tokens_details", None),
       })


   # 4) Optional: show the last log row we just wrote
   try:
       with LOG_PATH.open() as f:
           *_, last = f.read().strip().splitlines()
       print("Last log row:", last)
   except Exception:
       pass
copy

# pip install openai
import os, csv, time, pathlib, textwrap


# Minimal caller that returns the full response
def ask_capped(messages, *, model=MODEL, max_tokens=220, temperature=0):
   return client.chat.completions.create(
       model=model,
       messages=messages,
       temperature=temperature,
       max_tokens=max_tokens,
   )




# Budget guardrails
SESSION_BUDGET = 5.00   # dollars for this notebook/app session
session_spend = 0.0


def ask_budget(messages, *, model=MODEL, max_tokens=220, cost_cap=0.02):
   """
   Make a call with budget guardrails:
     - warn if this call exceeds `cost_cap`
     - warn if running `session_spend` exceeds `SESSION_BUDGET`
   """
   global session_spend


   # (Optional) keep inputs lean
   trimmed = trim_history(messages)


   # Make the request
   resp = ask_capped(trimmed, model=model, max_tokens=max_tokens)


   # Extract usage & cost
   u = resp.usage
   est = getattr(u, "estimated_cost", None)


   # Per-call warning
   if isinstance(est, (int, float)):
       if est > cost_cap:
           print(f"[WARN] Call ${est:.5f} exceeded cap ${cost_cap:.5f}. Consider lowering max_tokens or trimming context.")


       # Session accumulation + warning
       session_spend += est
       if session_spend > SESSION_BUDGET:
           print(f"[WARN] Session budget hit: ${session_spend:.2f} > ${SESSION_BUDGET:.2f}")


   else:
       # If the provider doesn’t return estimated_cost, still show tokens
       print("[INFO] Provider did not return estimated_cost; relying on token counts.")
       print(f"→ pt={getattr(u,'prompt_tokens',None)}, ct={getattr(u,'completion_tokens',None)}")


   return resp


# Example


msgs = [{"role":"user","content":"Write exactly 2 sentences about cached input pricing."}]
r = ask_budget(msgs, max_tokens=60, cost_cap=0.005)
print(textwrap.shorten(r.choices[0].message.content, width=160))
print(r)
print(session_spend)
copy