Update app.py

app.py

@@ -5,23 +5,27 @@ import uuid
 import numpy as np
 import faiss
 from sentence_transformers import SentenceTransformer
-from transformers import pipeline
 from PyPDF2 import PdfReader
 
-print("✅ App starting...")
+# Optional: if you had an NLI pipeline before, you can keep or remove it.
+try:
+    from transformers import pipeline as hf_pipeline
+    nli = hf_pipeline("text-classification", model="microsoft/deberta-large-mnli")
+    print("✅ NLI pipeline loaded.")
+except Exception as e:
+    nli = None
+    print("ℹ️ NLI pipeline not loaded (optional):", e)
+
 print("⏳ Loading SentenceTransformer model...")
 model = SentenceTransformer('all-MiniLM-L6-v2')
-print("✅ Model loaded.")
-
-print("⏳ Loading NLI pipeline...")
-nli = pipeline("text-classification", model="microsoft/deberta-large-mnli")
-print("✅ NLI pipeline loaded.")
+print("✅ Encoder loaded.")
 
 app = Flask(__name__)
 
 # ── base folders ───────────────────────────────────────────────────────────────
-BASE_UPLOADS = os.path.join(os.path.dirname(__file__), "uploads")
-BASE_RESULTS = os.path.join(os.path.dirname(__file__), "results")
+BASE_DIR = os.path.dirname(__file__)
+BASE_UPLOADS = os.path.join(BASE_DIR, "uploads")
+BASE_RESULTS = os.path.join(BASE_DIR, "results")
 os.makedirs(BASE_UPLOADS, exist_ok=True)
 os.makedirs(BASE_RESULTS, exist_ok=True)
 
@@ -39,10 +43,12 @@ def clear_uploads_folder():
             os.rmdir(path)
         else:
             os.remove(path)
+
 clear_uploads_folder()
 print("✅ Uploads folder cleared.")
 
-# runtime cache keyed by search-id → (paragraphs, embeddings, faiss-index)
+# runtime cache keyed by search-id → (paragraphs_norm, embeddings, faiss-index, spans)
+# spans[i] = (start_char, end_char) in the full merged text for paragraph i
 index_data = {}
 
 # ── helpers ────────────────────────────────────────────────────────────────────
@@ -56,38 +62,67 @@ def get_paths(sid: str):
     result_file = os.path.join(res_folder, "results.txt")
     return up_folder, res_folder, merged_file, result_file
 
-def extract_text(file_path):
-    if file_path.endswith('.txt'):
-        with open(file_path, 'r', encoding='utf-8') as f:
+def extract_text(file_path: str) -> str:
+    if file_path.lower().endswith('.txt'):
+        with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
             return f.read()
-    elif file_path.endswith('.pdf'):
+    if file_path.lower().endswith('.pdf'):
         reader = PdfReader(file_path)
-        full_text = " ".join(page.extract_text() for page in reader.pages if page.extract_text())
+        chunks = []
+        for page in reader.pages:
+            t = page.extract_text() or ""
+            chunks.append(t)
+        # Light cleanup: tighten excessive gaps, add blank lines between sentences
+        full_text = " ".join(chunks)
         full_text = re.sub(r'(?<=[.!?])\s{2,}', '\n\n', full_text)
-        full_text = re.sub(r'(?<=[a-z])\.\s+(?=[A-Z])', '.\n\n', full_text)
         full_text = re.sub(r'(\n\s*){2,}', '\n\n', full_text)
         return full_text
     return ""
 
+def split_paragraphs_with_spans(merged_text: str):
+    """
+    Split merged_text into logical 'paragraphs' based on blank lines,
+    returning normalized paragraphs for embedding AND exact spans (start,end)
+    in the original merged_text for highlighting/jumping.
+    """
+    sep = re.compile(r'(?:\r?\n\s*\r?\n)+', flags=re.MULTILINE)
+    paras_norm = []
+    spans = []
+    pos = 0
+    for m in sep.finditer(merged_text):
+        seg = merged_text[pos:m.start()]
+        norm = re.sub(r'\s+', ' ', seg).strip()
+        if len(norm.split()) > 4:  # keep only substantive chunks
+            paras_norm.append(norm)
+            spans.append((pos, m.start()))
+        pos = m.end()
+    # Tail
+    seg = merged_text[pos:]
+    norm = re.sub(r'\s+', ' ', seg).strip()
+    if len(norm.split()) > 4:
+        paras_norm.append(norm)
+        spans.append((pos, len(merged_text)))
+    return paras_norm, spans
+
 def rebuild_merged_and_index(sid: str):
-    """Re-embed everything for *this* search id."""
-    up_folder, _, merged_file, _ = get_paths(sid)
+    """Build merged.txt, paragraph embeddings, and spans for *this* search id."""
+    up_folder, res_folder, merged_file, _ = get_paths(sid)
 
     merged_text = ""
-    for filename in os.listdir(up_folder):
+    for filename in sorted(os.listdir(up_folder)):
         if filename.lower().endswith((".pdf", ".txt")):
-            merged_text += extract_text(os.path.join(up_folder, filename)) + "\n\n"
+            merged_text += extract_text(os.path.join(up_folder, filename)).rstrip() + "\n\n"
 
     with open(merged_file, "w", encoding='utf-8') as f:
         f.write(merged_text)
 
-    paras = re.split(r'\n\s*\n+', merged_text)
-    paras = [p.strip().replace('\n', ' ') for p in paras if len(p.strip().split()) > 4]
-    if not paras:
-        index_data[sid] = ([], None, None)
+    paras_norm, spans = split_paragraphs_with_spans(merged_text)
+
+    if not paras_norm:
+        index_data[sid] = ([], None, None, [])
         return
 
-    embed = model.encode(paras, batch_size=32, show_progress_bar=False)
+    embed = model.encode(paras_norm, batch_size=32, show_progress_bar=False)
     embed = np.asarray(embed)
     if embed.ndim == 1:
         embed = embed[np.newaxis, :]
@@ -95,7 +130,7 @@ def rebuild_merged_and_index(sid: str):
     idx = faiss.IndexFlatIP(embed.shape[1])
     idx.add(embed)
 
-    index_data[sid] = (paras, embed, idx)
+    index_data[sid] = (paras_norm, embed, idx, spans)
 
 # ── routes ─────────────────────────────────────────────────────────────────────
 @app.route("/", methods=["GET", "POST"])
@@ -105,7 +140,7 @@ def index():
         sid = str(uuid.uuid4())
 
     up_folder, _, _, _ = get_paths(sid)
-    paragraphs, embeddings, index_faiss = index_data.get(sid, ([], None, None))
+    paragraphs, embeddings, index_faiss, spans = index_data.get(sid, ([], None, None, []))
 
     uploaded_filenames = sorted(os.listdir(up_folder))
 
@@ -128,7 +163,7 @@ def index():
             faiss.normalize_L2(q_embed)
             D, I = index_faiss.search(q_embed, k=min(k, len(paragraphs)))
 
-            # Keep both the text and the FAISS paragraph index so the UI can fetch context.
+            # Keep the FAISS paragraph index so the UI can jump within full text.
             results = [{"idx": int(i), "text": paragraphs[i]} for i in I[0]]
 
             _, res_folder, _, result_file = get_paths(sid)
@@ -204,45 +239,40 @@ def ping():
 @app.route("/api/context")
 def api_context():
     """
-    Return an excerpt of the merged paragraphs centered on the requested paragraph index.
-    Query params: sid, idx (int), window (int, optional, default 3) – number of surrounding paras on each side.
+    Return FULL merged text plus the exact character span for the requested paragraph.
+    Query params: sid, idx (int)
+    Response: { merged: str, start: int, end: int, total_len: int }
     """
     sid = request.args.get("sid")
+    if not sid:
+        return jsonify(error="Missing sid"), 400
+
     try:
         idx = int(request.args.get("idx", "-1"))
     except (TypeError, ValueError):
         return jsonify(error="Bad idx"), 400
 
-    try:
-        window = int(request.args.get("window", "3"))
-    except (TypeError, ValueError):
-        window = 3
-
-    if not sid or idx < 0:
-        return jsonify(error="Missing sid or idx"), 400
-
-    paragraphs, _, _ = index_data.get(sid, (None, None, None))
-    if paragraphs is None:
+    paragraphs, _, _, spans = index_data.get(sid, (None, None, None, None))
+    if paragraphs is None or spans is None:
         return jsonify(error="No index for this sid. Upload files first."), 404
-    if idx >= len(paragraphs):
+
+    if not (0 <= idx < len(spans)):
         return jsonify(error="idx out of range"), 400
 
-    start = max(0, idx - window)
-    end = min(len(paragraphs), idx + window + 1)
-    context_paras = paragraphs[start:end]
-    center_local = idx - start  # where the highlighted paragraph sits in that slice
+    _, _, merged_file, _ = get_paths(sid)
+    if not os.path.exists(merged_file):
+        return jsonify(error="merged.txt not found"), 404
 
+    with open(merged_file, "r", encoding="utf-8") as f:
+        merged_text = f.read()
+
+    start, end = spans[idx]
     return jsonify(
-        paras=context_paras,
-        center=center_local,
+        merged=merged_text,
         start=start,
         end=end,
-        total=len(paragraphs)
+        total_len=len(merged_text)
     )
 
-#if __name__ == "__main__":
-#    from waitress import serve
-#    serve(app, host="0.0.0.0", port=9001, threads=4)
-
 if __name__ == "__main__":
     app.run(host="0.0.0.0", port=7860)