职位搜索算法优化：从 BM25 到向量检索的混合召回方案

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

from sentence_transformers import SentenceTransformer

model = SentenceTransformer('BAAI/bge-large-zh-v1.5')

# 职位向量化
job_text = f"{job['title']} {job['description']} {job['skills']}"
job_vector = model.encode(job_text)

# 存入 Elasticsearch（dense_vector 类型）
es.index(index="jobs_vector", id=job['id'], body={
    "vector": job_vector.tolist(),
    "job_id": job['id'],
})

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

def vector_search(query, top_k=50):
    query_vector = model.encode(query)
    result = es.search(index="jobs_vector", body={
        "knn": {
            "field": "vector",
            "query_vector": query_vector.tolist(),
            "k": top_k,
            "num_candidates": 100,
        }
    })
    return [(hit["_source"]["job_id"], hit["_score"]) for hit in result["hits"]["hits"]]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

def bm25_search(query, filters, top_k=50):
    result = es.search(index="jobs", body={
        "query": {
            "bool": {
                "must": [
                    {"multi_match": {
                        "query": query,
                        "fields": ["title^3", "skills^2", "description"],
                        "type": "best_fields"
                    }}
                ],
                "filter": filters
            }
        },
        "size": top_k
    })
    return [(hit["_id"], hit["_score"]) for hit in result["hits"]["hits"]]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

def rrf_fuse(bm25_results, vector_results, k=60):
    scores = {}
    # BM25 路
    for rank, (doc_id, _) in enumerate(bm25_results):
        scores[doc_id] = scores.get(doc_id, 0) + 1 / (k + rank + 1)
    # 向量路
    for rank, (doc_id, _) in enumerate(vector_results):
        scores[doc_id] = scores.get(doc_id, 0) + 1 / (k + rank + 1)
    # 按融合分数排序
    return sorted(scores.items(), key=lambda x: x[1], reverse=True)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

def rerank(doc_ids, user_profile, query):
    features = []
    for doc_id in doc_ids:
        job = get_job(doc_id)
        features.append({
            "bm25_score": get_bm25_score(query, doc_id),
            "vector_score": get_vector_score(query, doc_id),
            "job_quality": job["quality_score"],       # 职位质量分
            "company_reputation": job["company_score"], # 企业信誉分
            "user_match": match_score(user_profile, job), # 用户匹配度
            "freshness": (time.time() - job["created_at"]) / 86400, # 新鲜度
            "click_rate": job["click_rate"],           # 历史点击率
        })

    # LTR 模型预测最终分数
    scores = ltr_model.predict(features)
    ranked = sorted(zip(doc_ids, scores), key=lambda x: x[1], reverse=True)
    return [doc_id for doc_id, _ in ranked]

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20

def query_understanding(query):
    # 1. 拼写纠错
    corrected = spell_check(query)

    # 2. 同义词扩展
    expanded = synonym_expand(corrected)  # "前端开发" → "前端开发 OR Vue OR React OR web前端"

    # 3. 意图识别
    intent = classify_intent(corrected)  # 找工作 / 查薪资 / 看公司

    # 4. 实体抽取
    entities = extract_entities(corrected)  # 城市、薪资、经验等过滤条件

    return {
        "original": query,
        "corrected": corrected,
        "expanded": expanded,
        "intent": intent,
        "entities": entities,
    }