/**
 * GFS RAG Chunker
 * 
 * Splits documents into overlapping chunks for retrieval.
 * Uses semantic boundaries (headings, paragraphs) when possible.
 */

import crypto from 'crypto';
import { RAGDocument, RAGChunk, RAGDocumentCategory, DEFAULT_RAG_CONFIG } from './types';

/**
 * Split a document into chunks with overlap.
 */
export function chunkDocument(
  doc: RAGDocument,
  chunkSize: number = DEFAULT_RAG_CONFIG.chunkSize,
  chunkOverlap: number = DEFAULT_RAG_CONFIG.chunkOverlap
): RAGChunk[] {
  const chunks: RAGChunk[] = [];
  const content = doc.content.trim();

  if (!content) return [];

  // Try to split on semantic boundaries first
  const sections = splitOnSemanticBoundaries(content);

  let chunkIndex = 0;
  let buffer = '';

  for (const section of sections) {
    // If adding this section exceeds chunk size, flush
    if (buffer.length + section.length > chunkSize && buffer.length > 0) {
      chunks.push(createChunk(doc, buffer.trim(), chunkIndex));
      chunkIndex++;
      // Keep overlap from end of buffer
      buffer = buffer.slice(-chunkOverlap) + '\n';
    }
    buffer += section + '\n';
  }

  // Flush remaining buffer
  if (buffer.trim().length > 50) {
    chunks.push(createChunk(doc, buffer.trim(), chunkIndex));
  }

  return chunks;
}

/**
 * Split content on semantic boundaries (headings, double newlines, horizontal rules).
 */
function splitOnSemanticBoundaries(content: string): string[] {
  // Split on markdown headings, horizontal rules, and double newlines
  const boundaries = content.split(/(?=^#{1,4}\s)|(?=^---$)|(?=\n\n)/m);
  return boundaries.filter(s => s.trim().length > 0);
}

/**
 * Create a chunk with term vector for BM25-style retrieval.
 */
function createChunk(
  doc: RAGDocument,
  content: string,
  chunkIndex: number
): RAGChunk {
  const termVector = buildTermVector(content);

  return {
    id: `${doc.id}-chunk-${chunkIndex}`,
    documentId: doc.id,
    content,
    chunkIndex,
    termVector,
    source: doc.source,
    category: doc.category,
    title: doc.title,
    tags: doc.tags,
  };
}

/**
 * Build a term frequency vector for BM25-style matching.
 * Normalizes terms: lowercase, remove punctuation, filter stopwords.
 */
function buildTermVector(text: string): Record<string, number> {
  const vector: Record<string, number> = {};
  const words = text
    .toLowerCase()
    .replace(/[^a-z0-9\s-]/g, ' ')
    .split(/\s+/)
    .filter(w => w.length > 2 && !STOPWORDS.has(w));

  for (const word of words) {
    vector[word] = (vector[word] || 0) + 1;
  }

  return vector;
}

/**
 * Create a RAGDocument from a file path and content.
 */
export function createDocument(
  source: string,
  title: string,
  content: string,
  category: RAGDocumentCategory,
  tags: string[] = []
): RAGDocument {
  const contentHash = crypto.createHash('sha256').update(content).digest('hex').slice(0, 16);
  const id = `doc-${contentHash}`;

  return {
    id,
    source,
    category,
    title,
    content,
    tags,
    ingestedAt: new Date(),
    contentHash,
  };
}

const STOPWORDS = new Set([
  'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
  'of', 'with', 'by', 'from', 'is', 'are', 'was', 'were', 'be', 'been',
  'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
  'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'this',
  'that', 'these', 'those', 'it', 'its', 'not', 'no', 'nor', 'so',
  'yet', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such',
  'than', 'too', 'very', 'just', 'also', 'into', 'over', 'after',
]);
