0
Jump into concepts and practice - no test required
Recommended
from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity # Sample documents documents = [ "The cat sat on the mat.", "Dogs are great pets.", "Cats and dogs can live together.", "The quick brown fox jumps over the lazy dog.", "Pets bring joy to people." ] # Sample queries and their relevant document indices queries = ["cat on mat", "dogs as pets", "quick fox"] relevant_docs = [[0], [1, 2, 4], [3]] # Initialize TF-IDF vectorizer with stop words removal vectorizer = TfidfVectorizer(stop_words='english') # Fit on documents doc_vectors = vectorizer.fit_transform(documents) # Transform queries query_vectors = vectorizer.transform(queries) # Function to retrieve top documents for each query def retrieve_top_docs(query_vec, doc_vecs, top_k=3): similarities = cosine_similarity(query_vec, doc_vecs).flatten() top_indices = similarities.argsort()[::-1][:top_k] return top_indices, similarities[top_indices] # Evaluate precision and recall total_precision = 0 total_recall = 0 num_queries = len(queries) for i, query_vec in enumerate(query_vectors): top_docs, _ = retrieve_top_docs(query_vec, doc_vectors, top_k=3) retrieved_set = set(top_docs) relevant_set = set(relevant_docs[i]) true_positives = len(retrieved_set.intersection(relevant_set)) precision = true_positives / len(retrieved_set) if retrieved_set else 0 recall = true_positives / len(relevant_set) if relevant_set else 0 total_precision += precision total_recall += recall avg_precision = total_precision / num_queries * 100 avg_recall = total_recall / num_queries * 100 print(f"Average Precision: {avg_precision:.2f}%") print(f"Average Recall: {avg_recall:.2f}%")
Before: Precision ~60%, Recall ~50%
After: Precision 80%, Recall 73.33%
doc (case-insensitive)?doc.lower() and checks membership correctly. if doc.contains('apple'): uses a non-existent method contains. if 'Apple' == doc: compares whole string, not membership. if doc.find('apple') == -1: checks if find returns -1, which means not found, so logic is reversed.lower() + in for case-insensitive check [OK]lower() before checking membership [OK]find() return valuesdocuments = ['Apple pie recipe', 'Banana smoothie', 'apple tart'] query = 'apple' results = [doc for doc in documents if query.lower() in doc.lower()] print(results)
docs = ['Data science', 'Big Data', 'Machine learning'] query = 'data' results = [d for d in docs if d.find(query) != -1] print(results)
find behaviorfind method is case-sensitive, so searching 'data' in 'Data science' returns -1 (not found).find method is case-sensitive. 'Data science'.find('data') returns -1 because of uppercase 'D'. Similarly, 'Big Data'.find('data') returns -1. 'Machine learning' doesn't contain 'data'. So results is empty.find method is case-sensitive, so it misses 'Data science' -> Option Afind() is case-sensitive [OK]find() is case-sensitive; use lower() [OK]find() ignores casefind() return valuesdocs = ['Data Science is fun', 'I love machine learning', 'Deep learning and data']
setdefault to initialize lists and appends indices correctly with lowercase words. word_docs = {}
for i, doc in enumerate(docs):
for word in doc.split():
word_docs[word].append(i) misses initializing lists and ignores case. word_docs = {word: i for i, doc in enumerate(docs) for word in doc.lower().split()} creates a dict with last index only, not lists. word_docs = {}
for doc in docs:
for word in doc.lower().split():
word_docs[word] = doc overwrites values with document strings, not indices.setdefault and lowercase words for correct mapping [OK]setdefault to build lists for each word [OK]