import re, os

source = "path/to/your/xml/files/"

topoDict = {}

def updateDic(dic, key):
    if key in dic:
        dic[key] += 1
    else:
        dic[key]  = 1
    
def collectTaggedToponyms(xmlText, dic):
    xmlText = re.sub("\s+", " ", xmlText)
    date = re.search(r'<date value="([\d-]+)"', xmlText).group(1)
    count1, count2 = 0,0
    for t in re.findall(r"<placeName[^<]+</placeName>", xmlText):
        t = t.lower()
        if 'tgn,' in t:
            if re.search(r'reg="([^"]+)"', t):
                reg = re.search(r'reg="([^"]+)"', t).group(1)
            else:
                #print(t)
                reg = 0

            if re.search(r'key="([^"]+)"', t):
                key = re.search(r'key="([^"]+)"', t).group(1)
            else:
                #print(t)
                key = 0

            if reg == 0 or key == 0:
                count1 += 1
            else:
                count2 += 1
                keyNew = reg+"\t"+key
                updateDic(topoDict, keyNew)
##    if count1 >= 0:
##        print("%s: %d out of %d toponyms misstagged." % (date, count1, count2))


def collectRawToponyms(source):
    lof = os.listdir(source)
    lof = sorted(lof, reverse=False)
    counter = 0
        
    for f in lof:
        if f.startswith("dltext"): # fileName test        
            with open(source + f, "r", encoding="utf8") as f1:
                text = f1.read()                
                collectTaggedToponyms(text, topoDict)

    freqList = []
    thresh = 100
    for k,v in topoDict.items():
        if v >= thresh:
            freqList.append("%09d\t%s" % (v,k))
    print("Number of unique items with freq at least %d: %d" % (thresh, len(freqList)))

    # Number of unique items with freq at least 1: 9246
    # Number of unique items with freq at least 2: 5388
    # Number of unique items with freq at least 3: 4062
    # Number of unique items with freq at least 4: 3344
    # Number of unique items with freq at least 5: 2932

    freqList = "\n".join(sorted(freqList, reverse=True))
    with open("freqList.csv", "w", encoding="utf8") as f9:
        f9.write(freqList)
        

collectRawToponyms(source)

import re, os

def loadGeoData(fileName):
    dic = {}

    with open(fileName, "r", encoding="utf8") as f1:
        data = f1.read().split("\n")
        
        for d in data:
            d1 = d.split("\t")

            if len(d1) == 19:

                val = "\t".join([d1[1]+", "+d1[10], d1[4], d1[5]])
                test = d1[1].lower()

                if test in dic:
                    dic[test].append(val)
                else:
                    dic[test] = [val]
                
    return(dic)

geoDataFile = "./US/US.csv"
geoData = loadGeoData(geoDataFile)

def processResults(fileName):
    with open(fileName, "r", encoding="utf8") as f1:
        data = f1.read().split("\n")

        newData = []

        noResult = "\t".join(["NA", "NA", "NA"])

        for d in data:
            d1 = d.split("\t")

            if "," in d1[1]:
                test = d1[1].split(",")[0]
            else:
                test = d1[1]

            if test in geoData:
                for i in geoData[test]:
                    newData.append(d+"\t"+i)
            else:
                newData.append(d+"\t"+noResult)
            
    with open("matchedResults.csv", "w", encoding="utf8") as f9:
        f9.write("\n".join(newData))

processResults("freqList.csv")

import re, os

source = "/path/to/your/xml/files/"

topoDict = {}
def updateDic(dic, key):
    if key in dic:
        dic[key] += 1
    else:
        dic[key]  = 1

def loadMapLayer(fileName):
    dic = {}

    with open(fileName, "r", encoding="utf8") as f1:
        data = f1.read().split("\n")

        for d in data:
            d1 = d.split("\t")
            if d1[4] == "y":
                key = d1[2]
                val = "\t".join([d1[3], d1[5], d1[6]])
                dic[key] = val
                
    return(dic)

mapData = loadMapLayer("matchedResults_curated.csv")
   
def collectTaggedToponyms(xmlText, dic, dateFilter):
    xmlText = re.sub("\s+", " ", xmlText)
    date = re.search(r'<date value="([\d-]+)"', xmlText).group(1)

    for t in re.findall(r"<placeName[^<]+</placeName>", xmlText):
        t = t.lower()

        if re.search(r'"(tgn,\d+)', t):
            reg = re.search(r'"(tgn,\d+)', t).group(1)

            if reg in mapData:
                updateDic(topoDict, mapData[reg])

def collectMappableLayers(source, dateTest):
    lof = os.listdir(source)
    lof = sorted(lof, reverse=False)
    counter = 0
        
    for f in lof:
        if f.startswith("dltext"): # fileName test        
            with open(source + f, "r", encoding="utf8") as f1:
                text = f1.read()

                # date filter
                date = re.search(r'<date value="([\d-]+)"', text).group(1)
                if date.startswith(dateTest):
                    collectTaggedToponyms(text, topoDict, dateTest)

    freqList = []
    thresh = 1
    for k,v in topoDict.items():
        if v >= thresh:
            freqList.append("%09d\t%s\t%d" % (v,k, v//20))

    freqList = "\n".join(sorted(freqList, reverse=True))
    with open("Dispatch_Geo_%s.csv" % dateTest, "w", encoding="utf8") as f9:
        f9.write(freqList)
        

collectMappableLayers(source, "1863")

# Task: generate files for all years, all months, and all days
# - you can save results in subfolders;
# - or, better, in single files with dates in another column