实施相似性搜索算法

哈喽！大家好，很高兴又见面了，我是golang学习网的一名作者，今天由我给大家带来一篇《实施相似性搜索算法》，本文主要会讲到等等知识点，希望大家一起学习进步，也欢迎大家关注、点赞、收藏、转发! 下面就一起来看看吧！

获取数据

import pandas as pd


descripciones = [
        'all users must reset passwords every 90 days.',
        'passwords need to be reset by all users every 90 days.',
        'admin access should be restricted.',
        'passwords must change for users every 90 days.',
        'passwords must change for users every 80 days.'
    ]

# cargar el dataset
data = pd.dataframe({
    'rule_id': range(1, len(descripciones) + 1),
    'description': descripciones
})

词汇相似度

from sklearn.feature_extraction.text import tfidfvectorizer
from sklearn.metrics.pairwise import cosine_similarity

!
# vectorización de las descripciones con tf-idf
vectorizer = tfidfvectorizer().fit_transform(data['description'])

# calcular la matriz de similitud de coseno
cosine_sim_matrix = cosine_similarity(vectorizer)

# crear un diccionario para almacenar las relaciones sin duplicados
def find_related_rules(matrix, rule_ids, threshold=0.8):
    related_rules = {}
    seen_pairs = set()  # para evitar duplicados de la forma (a, b) = (b, a)

    for i in range(len(matrix)):
        related = []
        for j in range(i + 1, len(matrix)):  # j comienza en i + 1 para evitar duplicados
            if matrix[i, j] >= threshold:
                pair = (rule_ids[i], rule_ids[j])
                if pair not in seen_pairs:
                    seen_pairs.add(pair)
                    related.append((rule_ids[j], round(matrix[i, j], 2)))
        if related:
            related_rules[rule_ids[i]] = related

    return related_rules

# aplicar la función para encontrar reglas relacionadas
related_rules = find_related_rules(cosine_sim_matrix, data['rule_id'].tolist(), threshold=0.8)

# mostrar las reglas relacionadas
print("reglas relacionadas por similitud:")
for rule, relations in related_rules.items():
    print(f"rule {rule} es similar a:")
    for related_rule, score in relations:
        print(f"  - rule {related_rule} con similitud de {score}")

语义相似度

!pip install sentence-transformers
from sentence_transformers import SentenceTransformer, util


# Load the pre-trained model for generating embeddings
model = SentenceTransformer('all-MiniLM-L6-v2')

# Generate sentence embeddings for each rule description
embeddings = model.encode(data['Description'], convert_to_tensor=True)

# Compute the semantic similarity matrix
cosine_sim_matrix = util.cos_sim(embeddings, embeddings).cpu().numpy()

# Function to find related rules based on semantic similarity
def find_related_rules(matrix, rule_ids, threshold=0.8):
    related_rules = {}
    seen_pairs = set()  # To avoid duplicates of the form (A, B) = (B, A)

    for i in range(len(matrix)):
        related = []
        for j in range(i + 1, len(matrix)):  # Only consider upper triangular matrix
            if matrix[i, j] >= threshold:
                pair = (rule_ids[i], rule_ids[j])
                if pair not in seen_pairs:
                    seen_pairs.add(pair)
                    related.append((rule_ids[j], round(matrix[i, j], 2)))
        if related:
            related_rules[rule_ids[i]] = related

    return related_rules

# Apply the function to find related rules
related_rules = find_related_rules(cosine_sim_matrix, data['Rule_ID'].tolist(), threshold=0.8)

# Display the related rules
print("Reglas relacionadas por similitud semántica:")
for rule, relations in related_rules.items():
    print(f"Rule {rule} es similar a:")
    for related_rule, score in relations:
        print(f"  - Rule {related_rule} con similitud de {score}")

到这里，我们也就讲完了《实施相似性搜索算法》的内容了。个人认为，基础知识的学习和巩固，是为了更好的将其运用到项目中，欢迎关注golang学习网公众号，带你了解更多关于的知识点！