# Pulling Fed Rate Data

from bs4 import BeautifulSoup
import pandas as pd
import requests
import matplotlib.pyplot as plt

res = requests.get("https://www.federalreserve.gov/monetarypolicy/openmarket.htm")
soup = BeautifulSoup(res.content,'lxml')

list = {}
years = ['2021','2020','2019','2018','2017','2016','2015']
for x, y in zip(range(0,7), years):
    try:
        table = soup.find_all('table') 
        df = pd.read_html(str(table))[x]
        df['Date'] = str(y) + ' ' + df['Date'].astype(str)
        list['table ' + str(y)] = df
    except:
        pass

l2021 = pd.DataFrame(list['table 2021'])
l2020 = pd.DataFrame(list['table 2020'])
l2019 = pd.DataFrame(list['table 2019'])
l2018 = pd.DataFrame(list['table 2018'])
l2017 = pd.DataFrame(list['table 2017'])
l2016 = pd.DataFrame(list['table 2016'])
l2015 = pd.DataFrame(list['table 2015'])


fomc = pd.concat([l2021, l2020, l2019, l2018, l2017, l2016, l2015])
fomc['Increase'] = fomc['Increase'].replace('...','0')
fomc['Decrease'] = fomc['Decrease'].replace('...','0')
fomc['Decrease'] = fomc['Decrease'].replace('75-100','75')
fomc.reset_index(drop=True, inplace=True)
print(fomc)

fomc['delta'] = fomc['Increase'].astype(int) - fomc['Decrease'].astype(int)
fomc['year'] = pd.to_datetime(fomc['Date'])
fomc_1 = fomc.drop(['Date'], axis=1)
fomc_1['lower'] = fomc_1['Level (%)'].str.split('-').str[0].fillna(fomc_1['Level (%)'].str.split('-').str[1])
fomc_1['upper'] = fomc_1['Level (%)'].str.split('-').str[1].fillna(fomc_1['Level (%)'].str.split('-').str[0])
fomc_1['mid'] = (fomc_1['upper'].astype(float) + fomc_1['lower'].astype(float)) /2
print(fomc_1)

# Examining sentiment

import os
import re
import io
from io import StringIO
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfpage import PDFPage
import nltk
from nltk.stem import PorterStemmer
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.sentiment.vader import SentimentIntensityAnalyzer

path_1 = r'C:/Users/Toby/Desktop/PDF/FOMC_monetarypolicy/press_release'
os.chdir(path_1)
files = os.listdir()

names = [date for file in os.listdir('.') for date in re.findall("(\d{8})", file)]

sentiment = {}

for idx, (f, n) in enumerate(zip(files, names)):
    p  = os.path.join("c:", os.sep, "Users", "Toby", "Desktop", "PDF", "FOMC_monetarypolicy", "press_release", str(f))
    print(str(idx)+': '+p)

    def convert_pdf_to_txt(p):
        rsrcmgr = PDFResourceManager()
        retstr = io.StringIO()
        codec = 'utf-8'
        laparams = LAParams()
        device = TextConverter(rsrcmgr, retstr, codec=codec, laparams=laparams)
        fp = open(p, 'rb')
        interpreter = PDFPageInterpreter(rsrcmgr, device)
        password = ""
        maxpages = 0
        caching = True
        pagenos = set()

        for page in PDFPage.get_pages(fp, pagenos, maxpages=maxpages,
                                      password=password,
                                      caching=caching,
                                      check_extractable=True):
            interpreter.process_page(page)

        fp.close()
        device.close()
        text = retstr.getvalue()
        retstr.close()
        return text

    text = convert_pdf_to_txt(p)
    vader = SentimentIntensityAnalyzer()

    tokenized_words = word_tokenize(text)
    tags = nltk.pos_tag(tokenized_words)
    nouns = [word for word,pos in tags if (pos == 'NNP')]
    nouns = str(nouns).strip('[]')
    nouns = nouns.replace('\'','')

    arr = vader.polarity_scores(text)

    sentiment[n] = arr

df = pd.DataFrame.from_dict(sentiment)
df = df.T
df.reset_index(inplace=True)


df['Sentiment'] = df['neg']*100
df['Date'] = pd.to_datetime(df['index'])

print(df)
# print(fomc_1)

with plt.rc_context({'ytick.color':'black','xtick.color':'black'}):
    fig,ax = plt.subplots()
    ax.plot(fomc_1.year, fomc_1.mid, color='red')
    ax2 = ax.twinx()
    ax2.plot(df.Date, df.Sentiment, color='blue')
    ax.grid()
    ax.set_title('FOMC Press Release Sentiment vs. US Cash Rate')
    ax.set_ylabel('US Cash Rate (%)')
    ax2.set_ylabel('FOMC Neg Sentiment (%)')
    ax.legend(loc=0)
    plt.show()