# Simple text comparison software --- for education use
# Written by Craig Miller, February 2020
# Modified 16 February 2020
# Modified 8 April 2026
#
# Modified 13 April 2026:
#   1) Uses nltk word_tokenize and WordNetLemmatizer
#   2) Weights term values using TD - IDF
#
# To run:
#    1) Open this file in Python IDLE program
#    2) From the Run menu, select "Run Module"
#    3) At the Python Shell prompt, try out the example uses
#
# Example uses:
#
#    process_word("Cat")
#    process_word("Cats")
#    process_word("jumping")
#    process_word("don't")
#    process_word("ghostly")
#
#    t = analyze_file("docs/dracula.txt")
#    len(t)
#    t["vampire"]
#    t["ghost"]
#    summarize_table(t)
#
#    t2 = analyze_file("docs/jane_eyre.txt")
#    t2["ghost"]
#
#
#
#    analyze()       --> compare all txt files in the docs folder

import os
import math
import glob

import nltk
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from nltk import FreqDist

nltk.download("punkt")
nltk.download('punkt_tab')
nltk.download("wordnet")

lemmatizer = WordNetLemmatizer()
##text = "The students are running faster than they ran before"
##tokens = [lemmatizer.lemmatize(t.lower()) for t in word_tokenize(text)]
##freq = FreqDist(tokens)
##freq.most_common(10)


# global variables for configuration

# use these tokens for summarizing the counts
selected_list = ["dark", "stormy", "sad"]

##########
#### function definitions

def summarize_table(count_table):
    "summarize counts in table with most frequent and selected tokens"
    sorted_list = sorted(count_table.items(), key = lambda kv:(kv[1],kv[0]), reverse=True)
    limit = 30 if len(sorted_list) > 10 else len(sorted_list)
    print("Most frequent tokens in table...")
    for i in range(0, limit):
        print("{} --> {}".format(sorted_list[i][0], sorted_list[i][1]))
    print("Selected tokens in table...")
    for selection in selected_list:
        if selection in count_table:
            print("{} --> {}".format(selection, count_table[selection]))
        else:
            print("{} --> 0".format(selection))

def analyze_file(filename, word_set = set()):
    'returns the word count table for the given filename'
    print("Opening " + filename)
    fd = open(filename, "r", encoding="utf8")
    count_table = dict()
    data = fd.read()
    #print("splitting")

    # dataList = [lemmatizer.lemmatize(t.lower()) for t in word_tokenize(data)]
    dataList = []
    sset = set()
    for t in word_tokenize(data):
        root_t = lemmatizer.lemmatize(t.lower())
        dataList.append(root_t)
                        
    print("{} has {} words". format(filename, len(dataList)))

    for word in dataList:
        word_set.add(word)
        if word in count_table:
            count_table[word] += 1
        else:
            count_table[word] = 1
    fd.close()
    summarize_table(count_table)
    print()
    return count_table

def analyze(path = "docs"):
    '''
        read all texts from the docs folder,
        update term values using TF-IDF,
        report similarity comparisons, among all pairs
    '''
    
    doc_table = dict()
    word_set = set()
    os.chdir(path)
    fileList = glob.glob("[a-zA-Z0-9]*.txt")
    for fname in fileList:
        doc_table[fname] = analyze_file(fname, word_set)

    # for each word, count how many documents it appears in
    doc_count = dict.fromkeys(word_set, 0) # initialize all word keys to 0
    for word in word_set:
        for fname in fileList:
            if word in doc_table[fname]:
                doc_count[word] += 1

    # using document count for each word, weight by IDF
    for fname in fileList:
        for key in doc_table[fname]:
            doc_table[fname][key] *= math.log(len(fileList) / doc_count[key])
        print()
        print(f'Summarizing for {fname}')
        summarize_table(doc_table[fname])

    os.chdir('..')  # return to parent directory

    print()
    for fname in fileList:
        print("Comparing {} ... ".format(fname))
        for fname2 in fileList:
            sim = similarity(doc_table[fname], doc_table[fname2], word_set)
            print("  {:.2f} : {} ".format(sim, fname2))
        print()
                

def similarity(tableA, tableB, words = None):
    'return cosine similarity between tableA and tableB over all words'
    if not words:
        words = set(tableA.keys()).union(tableB.keys())
    
    ab = 0
    a2 = 0
    b2 = 0
    for w in words:
        ab += tableA.get(w, 0) * tableB.get(w, 0)
        a2 += tableA.get(w, 0) * tableA.get(w, 0)
        b2 += tableB.get(w, 0) * tableB.get(w, 0)
    return ab / (math.sqrt(a2) * math.sqrt(b2))
        




    
