from sklearn.cluster import KMeans
from sentence_transformers import SentenceTransformer

Embed all ticket summaries
model = SentenceTransformer('all-MiniLM-L6-v2')
embeddings = model.encode(ticket_summaries)

Find clusters (adjust n_clusters based on your data)
n_clusters = 20
kmeans = KMeans(n_clusters=n_clusters, random_state=42)
clusters = kmeans.fit_predict(embeddings)

Extract representative questions from each cluster
for i in range(n_clusters):
    cluster_tickets = [t for t, c in zip(tickets, clusters) if c == i]
    # Use centroid to find most representative ticket
    representative = cluster_tickets[0]

import openai

def generate_faq_from_tickets(tickets, max_faqs=15):
    """Extract FAQ pairs from support tickets using GPT-4."""

    # Group tickets by topic first (using embeddings)
    grouped_tickets = cluster_tickets(tickets)

    faqs = []
    for topic, topic_tickets in grouped_tickets.items():
        # Create context from actual tickets
        context = "\n".join([
            f"- {t['subject']}: {t['body'][:200]}"
            for t in topic_tickets[:5]
        ])

        response = openai.ChatCompletion.create(
            model="gpt-4",
            messages=[{
                "role": "system",
                "content": "Generate a FAQ question-answer pair from these support tickets. Be specific and accurate."
            }, {
                "role": "user",
                "content": f"Tickets:\n{context}\n\nGenerate one FAQ entry."
            }],
            temperature=0.3
        )

        faqs.append(parse_faq_response(response.choices[0].message.content))

    return faqs[:max_faqs]

def hybrid_faq_pipeline(tickets):
    # Step 1: Embed and cluster
    embeddings = embed_tickets(tickets)
    clusters = cluster_embeddings(embeddings, n_clusters=25)

    # Step 2: Extract topics using LDA or keywords
    topics = extract_topics(clusters)

    # Step 3: Generate FAQs with LLM
    faqs = []
    for topic in topics:
        context = build_context(topic.tickets)
        faq = llm_generate_faq(topic.name, context)
        faqs.append(faq)

    # Step 4: Deduplicate and rank by frequency
    faqs = deduplicate(faqs)
    faqs = rank_by_frequency(faqs, tickets)

    return faqs

import requests
import os

--- Zendesk ---
def fetch_zendesk_tickets(subdomain, email, api_token, days_back=180):
    """Fetch closed tickets from the last N days."""
    from datetime import datetime, timedelta
    since = (datetime.utcnow() - timedelta(days=days_back)).strftime("%Y-%m-%dT%H:%M:%SZ")
    url = f"https://{subdomain}.zendesk.com/api/v2/search.json"
    params = {
        "query": f"type:ticket status:closed created>{since}",
        "sort_by": "created_at",
        "sort_order": "desc",
        "per_page": 100,
    }
    tickets = []
    while url:
        resp = requests.get(url, params=params,
                            auth=(f"{email}/token", api_token))
        resp.raise_for_status()
        data = resp.json()
        tickets.extend(data.get("results", []))
        url = data.get("next_page")
        params = {}          # next_page already includes all params
    return tickets

--- Freshdesk ---
def fetch_freshdesk_tickets(domain, api_key, days_back=180):
    """Fetch resolved tickets using the filter endpoint."""
    url = f"https://{domain}.freshdesk.com/api/v2/tickets/filter"
    params = {"query": '"status:4"', "per_page": 100}   # status 4 = resolved
    tickets = []
    page = 1
    while True:
        resp = requests.get(url, params={params, "page": page},
                            auth=(api_key, "X"))
        if resp.status_code == 404:
            break
        resp.raise_for_status()
        batch = resp.json().get("results", [])
        if not batch:
            break
        tickets.extend(batch)
        page += 1
    return tickets

def preprocess_tickets(tickets):
    cleaned = []
    for ticket in tickets:
        # Remove personal information
        text = anonymize(ticket['body'])

        # Normalize formatting
        text = normalize_whitespace(text)

        # Filter out noise (auto-responses, internal notes)
        if not is_autoresponse(ticket) and not is_internal(ticket):
            cleaned.append({
                'subject': ticket['subject'],
                'body': text,
                'category': ticket.get('category', 'general')
            })

    return cleaned

def validate_faq(faq, existing_faqs):
    # Check for duplicates
    for existing in existing_faqs:
        if cosine_similarity(faq.embedding, existing.embedding) > 0.85:
            return False, "Duplicate"

    # Check question is answerable
    if len(faq.answer) < 50:
        return False, "Answer too short"

    # Check relevance to product
    if not is_relevant_to_product(faq.question):
        return False, "Off-topic"

    return True, "Valid"

def score_faq(faq, ticket_cluster):
    frequency   = len(ticket_cluster)
    escalations = sum(1 for t in ticket_cluster if t.get("escalated"))
    avg_handle  = sum(t.get("handle_time_minutes", 0) for t in ticket_cluster) / max(frequency, 1)

    # Weighted composite score
    score = (frequency * 1.0) + (escalations * 3.0) + (avg_handle * 0.5)
    return score

from datetime import datetime, timedelta

def incremental_faq_update(existing_faqs, since_date):
    new_tickets = fetch_tickets(since=since_date)
    if not new_tickets:
        return existing_faqs, []

    new_embeddings = embed_tickets(new_tickets)
    changed = []

    for i, faq in enumerate(existing_faqs):
        cluster_new = [t for t, e in zip(new_tickets, new_embeddings)
                       if cosine_similarity(e, faq.centroid_embedding) > 0.75]
        growth_rate = len(cluster_new) / max(faq.ticket_count, 1)
        if growth_rate > 0.20:
            updated = llm_generate_faq(faq.topic, cluster_new)
            existing_faqs[i] = updated
            changed.append(updated)

    return existing_faqs, changed

import json
from datetime import datetime, timedelta
from collections import Counter
import openai

class FAQPipeline:
    """Complete FAQ generation pipeline from support tickets"""

    def __init__(self, api_key, min_ticket_frequency=5):
        openai.api_key = api_key
        self.min_ticket_frequency = min_ticket_frequency

    def load_tickets(self, filepath, days_back=90):
        """Load recent support tickets from JSON"""
        cutoff_date = datetime.now() - timedelta(days=days_back)

        with open(filepath) as f:
            all_tickets = json.load(f)

        recent = [
            t for t in all_tickets
            if datetime.fromisoformat(t['created_at']) > cutoff_date
        ]

        return recent

    def cluster_tickets(self, tickets, n_clusters=25):
        """Group tickets by topic using embeddings"""
        from sentence_transformers import SentenceTransformer
        from sklearn.cluster import KMeans

        model = SentenceTransformer('all-MiniLM-L6-v2')

        # Embed ticket summaries
        summaries = [t['subject'] for t in tickets]
        embeddings = model.encode(summaries)

        # Cluster
        kmeans = KMeans(n_clusters=min(n_clusters, len(tickets)), random_state=42)
        clusters = kmeans.fit_predict(embeddings)

        # Group tickets by cluster
        grouped = {}
        for ticket, cluster_id in zip(tickets, clusters):
            if cluster_id not in grouped:
                grouped[cluster_id] = []
            grouped[cluster_id].append(ticket)

        return grouped

    def generate_faq_for_cluster(self, cluster_tickets, topic_name):
        """Generate a single FAQ entry from clustered tickets"""
        context = "\n".join([
            f"- {t['subject']}: {t['body'][:300]}"
            for t in cluster_tickets[:10]
        ])

        response = openai.ChatCompletion.create(
            model="gpt-4",
            messages=[{
                "role": "system",
                "content": "Generate a clear, concise FAQ question-answer pair."
            }, {
                "role": "user",
                "content": f"""Topic: {topic_name}

Tickets:
{context}

Generate:
1. A natural question customers might ask
2. A complete answer (2-3 sentences max)
3. A link label if external docs exist

Format as JSON - {{"question": "...", "answer": "...", "link": "..."}}"""
            }],
            temperature=0.3,
            max_tokens=200
        )

        return json.loads(response.choices[0].message.content)

    def deduplicate_faqs(self, faqs):
        """Remove duplicate or very similar FAQs"""
        from sentence_transformers import util

        model = SentenceTransformer('all-MiniLM-L6-v2')
        embeddings = model.encode([f['question'] for f in faqs])

        unique_faqs = []
        seen_indices = set()

        for i, faq in enumerate(faqs):
            if i in seen_indices:
                continue

            # Find similar questions
            similarities = util.pytorch_cos_sim(embeddings[i], embeddings)
            similar_indices = (similarities[0] > 0.85).nonzero(as_tuple=True)[0]

            # Keep highest frequency one
            for idx in similar_indices:
                seen_indices.add(int(idx))

            unique_faqs.append(faq)

        return unique_faqs

    def rank_by_impact(self, faqs, tickets):
        """Rank FAQs by frequency in tickets"""
        for faq in faqs:
            # Count ticket matches for this FAQ
            matching_count = sum(
                1 for t in tickets
                if faq['question'].lower() in t['subject'].lower()
            )
            faq['impact_score'] = matching_count

        return sorted(faqs, key=lambda f: f['impact_score'], reverse=True)

    def run(self, tickets_file, output_file, n_faqs=15):
        """Run complete pipeline"""
        print("Loading tickets...")
        tickets = self.load_tickets(tickets_file)

        print(f"Clustering {len(tickets)} tickets...")
        clusters = self.cluster_tickets(tickets)

        print("Generating FAQ entries...")
        faqs = []
        for cluster_id, cluster_tickets in clusters.items():
            if len(cluster_tickets) < self.min_ticket_frequency:
                continue

            topic = f"Topic {cluster_id}"
            faq = self.generate_faq_for_cluster(cluster_tickets, topic)
            faqs.append(faq)

        print("Deduplicating...")
        faqs = self.deduplicate_faqs(faqs)

        print("Ranking by impact...")
        faqs = self.rank_by_impact(faqs, tickets)

        print(f"Writing {min(len(faqs), n_faqs)} FAQs...")
        output = {
            "generated_at": datetime.now().isoformat(),
            "total_tickets_analyzed": len(tickets),
            "faqs": faqs[:n_faqs]
        }

        with open(output_file, 'w') as f:
            json.dump(output, f, indent=2)

        return output

Usage
pipeline = FAQPipeline(api_key="sk-...")
result = pipeline.run(
    "support_tickets.json",
    "generated_faqs.json",
    n_faqs=20
)

sync-to-zendesk.py
import requests
import json

ZENDESK_API_KEY = os.getenv('ZENDESK_API_KEY')
ZENDESK_SECTION_ID = 123456

def sync_faqs_to_zendesk(faqs_file):
    """Publish generated FAQs to Zendesk"""
    with open(faqs_file) as f:
        faqs = json.load(f)['faqs']

    for faq in faqs[:20]:  # Limit to 20 for API quota
        article = {
            "article": {
                "title": faq['question'],
                "body": faq['answer'],
                "section_id": ZENDESK_SECTION_ID,
                "draft": False,
            }
        }

        response = requests.post(
            f"https://your-subdomain.zendesk.com/api/v2/help_center/articles.json",
            json=article,
            headers={
                "Authorization": f"Bearer {ZENDESK_API_KEY}",
                "Content-Type": "application/json"
            }
        )

        if response.status_code == 201:
            print(f"Created: {faq['question']}")
        else:
            print(f"Failed: {response.status_code} - {response.text}")

def measure_faq_impact(before_tickets, after_tickets, faq_published_date):
    """Compare support volume before and after FAQ publication"""
    before = [t for t in before_tickets if t['created_at'] < faq_published_date]
    after = [t for t in after_tickets if t['created_at'] >= faq_published_date]

    reduction = (len(before) - len(after)) / len(before) * 100

    print(f"Tickets before FAQ: {len(before)}")
    print(f"Tickets after FAQ: {len(after)}")
    print(f"Reduction - {reduction:.1f}%")

    # Identify which FAQ topics have highest impact
    common_before = Counter([t['category'] for t in before]).most_common(10)
    common_after = Counter([t['category'] for t in after]).most_common(10)

    print("\nMost reduced categories:")
    for (category, before_count), (_, after_count) in zip(common_before, common_after):
        reduction_pct = ((before_count - after_count) / before_count * 100)
        print(f"  {category}: {reduction_pct:.0f}% reduction")