# Simple text comparison software --- for education use
# Written by Craig Miller, February 2020
# Modified 16 February 2020
# Modified 8 April 2026
#
# To run:
#    1) Open this file in Python IDLE program
#    2) From the Run menu, select "Run Module"
#    3) At the Python Shell prompt, try out the example uses
#
# Example uses:
#
#    process_word("Cat")
#    process_word("Cats")
#    process_word("jumping")
#    process_word("don't")
#    process_word("ghostly")
#
#    t = analyze_file("docs/dracula.txt")
#    len(t)
#    t["vampire"]
#    t["ghost"]
#    summarize_table(t)
#
#    t2 = analyze_file("docs/jane_eyre.txt")
#    t2["ghost"]
#
#
#
#    analyze()       --> compare all txt files in the docs folder

import os
import math
import glob

# global variables for configuration

include_list = None
# uncomment statement below to *only* include selected words --- all other words are ignored
# include_list = ["I", "you", "me", "they"]

stem_list = ["s", "ing", "ed", "ly"]
stop_list = ["the", "a", "to", "and", "of", "in", "it", "that", "for", "so", 
             "am", "is", "are", "was", "be", "were", "been", "being", "from", "by",
             "as", "with", "not", "was", "on", "at", "its", "but", "have", "has",
             "had"]

# use these tokens for summarizing the counts
selected_list = ["dark", "stormy", "sad"]

##########
#### function definitions

def process_word(word):
    "process word and return a new word so that it matches similar words"
    word = strip_punctuation(word)
    word = word.lower()
    word = stem(word)
    return word

def strip_punctuation(word):
    "return word with only alphabetic characters"
    new_word = ""
    for chr in word:
        if chr.isalpha():
            new_word = new_word + chr
    return new_word

def stem(word):
    "creates a new word from the given word without suffix"
    for suffix in stem_list:
        if word.endswith(suffix) and len(word) / len(suffix) > 3:
            return word[0:-len(suffix)]
    return word
            
        
def count_word(table, word):
    'for the word entry in the table, increment its count or init to 1'
    if include_list == None:
        if word in stop_list:
            return
        word = process_word(word)
        if word == "":
            return
    else:
        if word not in include_list:
            return
    if word in table:
        table[word] += 1
    else:
        # initialize count of word to 1
        table[word] = 1

def summarize_table(count_table):
    "summarize counts in table with most frequent and selected tokens"
    sorted_list = sorted(count_table.items(), key = lambda kv:(kv[1],kv[0]), reverse=True)
    limit = 10 if len(sorted_list) > 10 else len(sorted_list)
    print("Most frequent tokens in table...")
    for i in range(0, limit):
        print("{} --> {}".format(sorted_list[i][0], sorted_list[i][1]))
    print("Selected tokens in table...")
    for selection in selected_list:
        if selection in count_table:
            print("{} --> {}".format(selection, count_table[selection]))
        else:
            print("{} --> 0".format(selection))

def analyze_file(filename, word_set = set()):
    'returns the word count table for the given filename'
    print("Opening " + filename)
    fd = open(filename, "r", encoding="utf8")
    count_table = dict()
    data = fd.read()
    #print("splitting")
    dataList = data.split()
    print("{} has {} words". format(filename, len(dataList)))

    for word in dataList:
        word_set.add(word)
        count_word(count_table, word)
    fd.close()
    summarize_table(count_table)
    print()
    return count_table

def analyze(path = "docs"):
    '''read all texts from the docs folder, report similarity comparisons
        among all pairs'''
    
    doc_table = dict()
    word_set = set()
    os.chdir(path)
    fileList = glob.glob("[a-zA-Z0-9]*.txt")
    for fname in fileList:
        doc_table[fname] = analyze_file(fname, word_set)

    os.chdir('..')  # return to parent directory

    for fname in fileList:
        print("Comparing {} ... ".format(fname))
        for fname2 in fileList:
            sim = similarity(doc_table[fname], doc_table[fname2], word_set)
            print("  {:.2f} : {} ".format(sim, fname2))
        print()
                

def similarity(tableA, tableB, words = None):
    'return cosine similarity between tableA and tableB over all words'
    if not words:
        words = set(tableA.keys()).union(tableB.keys())
    
    ab = 0
    a2 = 0
    b2 = 0
    for w in words:
        ab += tableA.get(w, 0) * tableB.get(w, 0)
        a2 += tableA.get(w, 0) * tableA.get(w, 0)
        b2 += tableB.get(w, 0) * tableB.get(w, 0)
    return ab / (math.sqrt(a2) * math.sqrt(b2))
        




    
