搜索查询的向量聚类：使用 ChromaDB、嵌入（embeddings）和 Python 构建您自己的系统

import os
from openai import OpenAI

# Initialize the client
client_ai = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))

# Our raw dataset
queries = ["buy iphone 15", "iphone 15 pro price", "apple phone repair"]

# 1. Vectorize queries through OpenAI
response = client_ai.embeddings.create(
    input=queries,
    model="text-embedding-3-small"
)

# The result is a set of ready-made multidimensional vectors
embeddings = [data.embedding for data in response.data]

from sentence_transformers import SentenceTransformer

print("⏳ 1. Loading stable BGE-m3 model...")
model = SentenceTransformer('BAAI/bge-m3')

queries = ["buy iphone 15", "iphone 15 pro price", "apple phone repair"]

print(f"⏳ 2. Vectorizing {len(queries)} queries...")
embeddings = model.encode(queries, normalize_embeddings=True)

print("\n✅ Done! Let's look at the result:")
print(f"📊 Data dimensions: {embeddings.shape}")
print("🔍 Vector for the phrase 'buy iphone 15':")
vector_preview = [round(float(num), 4) for num in embeddings[0][:5]]
print(f"🔢 {vector_preview} ... and {len(embeddings[0]) - 5} more numbers.")

import chromadb

# 1. Initialize the local database (creates the semantic_db folder)
client = chromadb.PersistentClient(path="./semantic_db")

# 2. Create the collection
collection = client.get_or_create_collection(
    name="search_queries",
    metadata={"hnsw:space": "cosine"}
)

# 3. Load our vectors and query texts
collection.add(
    embeddings=embeddings.tolist(), # vectors from our local BGE-m3 model
    documents=queries,
    ids=["id_1", "id_2", "id_3"]
)
print("✅ Data saved to the database!\n")

# ==========================================
# SEARCH MAGIC: let's check how the database understands meaning
# ==========================================
test_phrase = "how much does the new iphone cost"
print(f"Searching the database for the phrase: '{test_phrase}'")

# Convert the test phrase into a vector using the same model
test_embedding = model.encode([test_phrase], normalize_embeddings=True)

# Ask the database to find the two most similar options
results = collection.query(
    query_embeddings=test_embedding.tolist(),
    n_results=2
)

print(f"Closest query: {results['documents'][0][0]} (Distance: {results['distances'][0][0]:.4f})")
print(f"Second closest: {results['documents'][0][1]} (Distance: {results['distances'][0][1]:.4f})")

import chromadb
import numpy as np
from sklearn.cluster import DBSCAN

print("⏳ 1. Connecting to the vector database...")
client = chromadb.PersistentClient(path="./semantic_db")

# Extract the collection with the vectors (use your own name)
collection = client.get_collection(name="search_queries") 

# Extract all query texts and their mathematical vectors
data = collection.get(include=["documents", "embeddings"])
documents = data["documents"]
embeddings = np.array(data["embeddings"])

print(f"✅ Queries extracted from the database: {len(documents)}")

# ==========================================
# CLUSTERING (DBSCAN)
# ==========================================
print("⏳ 2. Starting the DBSCAN algorithm...")

# SETTINGS:
# eps = 0.15 (Allowed cosine distance. The smaller it is, the stricter the clusters);
# min_samples = 2 (Minimum of 2 queries to create a group);
# metric="cosine" (We explicitly specify that we measure angles between vectors, not linear distance).
dbscan = DBSCAN(eps=0.15, min_samples=2, metric="cosine")

# Run the grouping
labels = dbscan.fit_predict(embeddings)

# ==========================================
# OUTPUT RESULTS
# ==========================================
# The algorithm assigned each query a group number (0, 1, 2...). 
# If a query is recognized as noise (Outlier), it receives the label -1.

clusters = {}
outliers = []

for doc, label in zip(documents, labels):
    if label == -1:
        outliers.append(doc)
    else:
        if label not in clusters:
            clusters[label] = []
        clusters[label].append(doc)

print("=== GROUPING RESULTS ===")
for cluster_id, docs in clusters.items():
    print(f"\n Cluster #{cluster_id} (Queries: {len(docs)})")
    for d in docs:
        print(f"  - {d}")

if outliers:
    print(f"\n Outliers/Noise (Queries: {len(outliers)})")
    for out in outliers:
        print(f"  - {out}")

queries = [
    # Informational
    "what are antidetect browsers for",
    "what is an antidetect browser",
    "how antidetect browsers work",
    "antidetect browsers comparison",
    
    # Commercial
    "buy an anti-detect browser",
    "buy proxies for an anti-detect browser",
    "anti-detect browser trial",
    
    # Download
    "download anti-detect browser octo browser",
    "octo browser anti-detect browser download",
    "octo anti-detect browser download",
    
    # Queries on a different topic
    "ford everest 2024 review", 
    "buy used ford everest",
    
    # Noise
    "weather in pattaya in May",
    "tom yum soup recipe"
]

from openai import OpenAI

# 1. INITIALIZATION AND KEY
# Insert your actual API key here
client_ai = OpenAI(api_key="sk-YOUR_OPENAI_KEY")

# 2. OUR DATA (Result of hybrid clustering)
clusters = {
    0: [
       "what are anti-detect browsers for",
       "what is an anti-detect browser",
       "how anti-detect browsers work",
       "anti-detect browsers comparison",
    ],
    1: [
        "buy an anti-detect browser",
        "buy proxies for an anti-detect browser",
        "anti-detect browser trial"
    ],
    2: [
        "download anti-detect browser octo browser",
        "octo browser anti-detect browser download",
        "octo anti-detect browser download"
    ]
}

# System prompt (set rules for the model)
prompt = """You are an expert SEO specialist. Analyze the following cluster of search queries. Determine the primary user intent and generate one highly relevant H1 title for a future category page or article. Return ONLY the title, without any additional text, quotes or explanations."""

print("⏳ Sending clusters to GPT-4o-mini for automatic naming...\n")

# 3. Iterate through all clusters
for cluster_id, queries in clusters.items():
    # Send the queries from the current cluster to the API
    response = client_ai.chat.completions.create(
        model="gpt-4o-mini",
        messages=[
            {"role": "system", "content": prompt},
            {"role": "user", "content": "\n".join(queries)} # Combine the queries into a single text
        ],
        temperature=0.3
    )
    
    # Get the response
    h1_title = response.choices[0].message.content
    
    # Print the result to the console
    print(f"Cluster #{cluster_id}")
    print(f"Phrases: {', '.join(queries)}")
    print(f"Generated H1: {h1_title}\n")

Cluster #0
   Phrases: what are anti-detect browsers for, what is an anti-detect browser, how anti-detect browsers work
   Generated H1: What are anti-detect browsers used for

Cluster #1
   Phrases: buy an anti-detect browser, buy proxies for an anti-detect browser, anti-detect browser trial
   Generated H1: Best anti-detect browsers and proxies for safe browsing

Cluster #2
   Phrases: download anti-detect browser octo browser, octo browser anti-detect browser download, octo anti-detect browser download
   Generated H1: Download anti-detect browser Octo Browser