import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import nltk

d_Making_it_some_what_dynamic = """
Making it some what dynamic..."""
d_Making_it_properly_dynamic = """
Making it properly dynamic..."""
d_detection_script_debug  = """
What's new detection script debug and fix..."""
d_Website_server_migration= """
Website server migration..."""
d_Creating_mdbook_summary_maker = """
Creating mdbook-summary-maker - a summary.md automation tool..."""
d_creating_this_website = r"""
Creating this website..."
""

documents = [d_Creating_mdbook_summary_maker, d_creating_this_website, d_detection_script_debug, d_Making_it_properly_dynamic, d_Making_it_some_what_dynamic, d_Website_server_migration]

vrizer = CountVectorizer(stop_words="english")
vrizer.fit(documents)
X = vrizer.transform(documents)
print(X.shape)
dtm = pd.DataFrame(X.toarray(),
        	columns=vrizer.get_feature_names_out())
dtm

cosine_sim_matrix_for_antzed = cosine_similarity(dtm)
cosine_sim_matrix_for_antzed.shape 
document_names = [
    "d_Creating_mdbook_summary_maker",
    "d_creating_this_website",
    "d_detection_script_debug",
    "d_Making_it_properly_dynamic",
    "d_Making_it_some_what_dynamic",
    "d_Website_server_migration"
]

# Initialize variables to store the maximum similarity and the document indices
max_similarity = 0
doc_index_1 = 0
doc_index_2 = 0

# Iterate over the matrix to find the pair with the highest similarity
for i in range(6):
    for j in range(i+1, 6):  # Ensure no self-comparison
        if cosine_sim_matrix_for_antzed[i, j] > max_similarity:
            max_similarity = cosine_sim_matrix_for_antzed[i, j]
            doc_index_1 = i
            doc_index_2 = j

# Print the document names with the highest similarity and their similarity score
print(f"The documents with the highest similarity are \"{document_names[doc_index_1]}\" and \"{document_names[doc_index_2]}\" with a similarity score of {max_similarity:.4f}.")

import gensim

import nltk
from nltk.stem import WordNetLemmatizer
from nltk.corpus import wordnet
from gensim.utils import simple_preprocess
from nltk.corpus import stopwords

nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
nltk.download('punkt')
nltk.download('stopwords')

stop_words = set(stopwords.words('english'))
lemmatizer = WordNetLemmatizer()

from gensim.utils import simple_preprocess
from gensim.parsing.preprocessing import preprocess_string

def get_wordnet_pos(word):
    """Map NLTK's part of speech tags to wordnet's"""
    tag = nltk.pos_tag([word])[0][1][0].upper()
    tag_dict = {"J": wordnet.ADJ,
                "N": wordnet.NOUN,
                "V": wordnet.VERB,
                "R": wordnet.ADV}
    return tag_dict.get(tag, wordnet.NOUN)


def preprocess_text(text):
    tokens = simple_preprocess(text, deacc=True)  # Tokenize into words
    tokens_no_stops = [token for token in tokens if token not in stop_words]
    lemmatized_tokens = [lemmatizer.lemmatize(token, get_wordnet_pos(token)) for token in tokens_no_stops]
    return lemmatized_tokens

# Tokenize and preprocess all documents
processed_sentences = [preprocess_text(doc) for doc in documents]

# Flatten the list if your model expects a list of sentences as input
flat_list_of_words = [word for sublist in processed_sentences for word in sublist]

print(flat_list_of_words[:5])

from gensim.models import Word2Vec

# Now that we have our sentences preprocessed and lemmatized, we train the model
model = Word2Vec([flat_list_of_words], vector_size=100, window=5, min_count=1, workers=4)

# Save the model for later use
model.save("lemmatized_word2vec_model.model")

model.wv.most_similar('rust', topn=10)

model.wv.most_similar('dynamic', topn=10)

model.wv.most_similar('website', topn=10)

model.wv.most_similar('antzed', topn=10)

model.wv.most_similar('problem', topn=10)