4_subject.py

import pandas as pd
import numpy as np
import csv
import spacy
import json
import re

from datetime import datetime, date
import datetimerange

d = pd.read_csv('data/final_data_binary_decision.csv')
d['Message_x'] = d['Message_x'].astype(str)
print(len(d))

d_sample = d[d['preds_binary'] == 1]
# Contrary to training data, the creation date is in a different format, so adapt
d_sample['Post Created Date'] = d_sample['Post Created Date'].apply(str)
d_sample['Post Created Date'] = d_sample['Post Created Date'].str.split(' ')
d_sample['Post Created Date'] = d_sample['Post Created Date'].apply(lambda x: x[0])
print(len(d_sample))


def get_entities(text):
        """ 
        Helper function to get the words and the entities
        """
        list_entities = []
        relevant_entities = ['ORG', 'NORP', 'PERSON', 'POSITION']
        for word in text.ents:
            if word.label_ in relevant_entities:
                list_entities.append({'entity': word.text, 'label': word.label_})
        
        # I need to remove parentheses from the entities. If they end up here, this means the model failed anyways.
        # The parentheses confused the code in a later part
        for entity in list_entities:
                entity['entity'] = entity['entity'].replace('(', '')
                entity['entity'] = entity['entity'].replace(')', '')
        
        return list_entities
    

def apply_ner():
        """ 
        A function that applies the custom-made NER model to the text. The code used to make this model can be found at
        "6_gazetteers_data_Yannick.ipynb".
        """
        
        # Define relevant entities for this task:
        relevant_entities = ['ORG', 'NORP', 'PERSON', 'POSITION'] # (NORP = "nationalities or religious or political groups")
        
        # Load in spacy:
        nlp = spacy.load("code/spacy_model_subject")

        # Apply NER:
        d_sample['spacy'] = d_sample['Message_x'].apply(nlp) 
        d_sample['ners'] = d_sample['spacy'].apply(get_entities)

        return d_sample


def create_periods(start, end):
        """ 
        Helper function that creates a period given a start and end datetime object
        """
        period = datetimerange.DateTimeRange(start, end)
        return period
    
def select_database(date):
        """ 
        A function that opens the correct database based on the date the post was created.
        These files are not currently included in this repository. The files contain the names 
        and party affiliation of every politician who was a member of the Flemish/Federal Parliament/Government at a certain time.
        """

        # Convert the date to datetime object
        dt = datetime.strptime(date, '%Y-%m-%d')

        # Create ranges of periods, based on metadata in excel from August
        p0_range = create_periods(datetime(2009, 6, 30), datetime(2010, 6, 13)) 
        p1_range = create_periods(datetime(2010, 6, 14), datetime(2011, 12, 5))
        p2_range = create_periods(datetime(2011, 12, 6), datetime(2014, 5, 25))
        p3_range = create_periods(datetime(2014, 5, 26), datetime(2014, 7, 24))
        p4_range = create_periods(datetime(2014, 7, 25), datetime(2014, 10, 10))
        p5_range = create_periods(datetime(2014, 10, 11), datetime(2016, 9, 20))
        p6_range = create_periods(datetime(2016, 9, 21), datetime(2018, 12, 8))
        p7_range = create_periods(datetime(2018, 12, 9), datetime(2019, 5, 26))
        p8_range = create_periods(datetime(2019, 5, 27), datetime(2019, 7, 1))
        p9_range = create_periods(datetime(2019, 7, 2), datetime(2019, 10, 1))
        p10_range = create_periods(datetime(2019, 10, 2), datetime(2019, 10, 26))
        p11_range = create_periods(datetime(2019, 10, 27), datetime(2020, 3, 16))
        p12_range = create_periods(datetime(2020, 3, 17), datetime(2020, 10, 1))
        p13_range = create_periods(datetime(2020, 10, 2), datetime(2022, 6, 28))
        p14_range = create_periods(datetime(2022, 6, 29), datetime(2022, 11, 17)) 
        p15_range = create_periods(datetime(2022, 11, 18), datetime(2022, 12, 16))
        p16_range = create_periods(datetime(2022, 12, 17), datetime(2023, 10, 21))
        p17_range = create_periods(datetime(2023, 10, 22), datetime(2023, 11, 7))
        p18_range = create_periods(datetime(2023, 11, 8), datetime(2023, 12, 6))
        p19_range = create_periods(datetime(2023, 12, 7), datetime(2024, 1, 31))

        # Get paths to related dataframes
        path_dfs = '/path/to/data/pol-actors-updated/'
        

        p0 = path_dfs + 'period0.csv'
        p1 = path_dfs + 'period1.csv'
        p2 = path_dfs + 'period2.csv'
        p3 = path_dfs + 'period3.csv'
        p4 = path_dfs + 'period4.csv'
        p5 = path_dfs + 'period5.csv'
        p6 = path_dfs + 'period6.csv'
        p7 = path_dfs + 'period7.csv'
        p8 = path_dfs + 'period8.csv'
        p9 = path_dfs + 'period9.csv'
        p10 = path_dfs + 'period10.csv'
        p11 = path_dfs + 'period11.csv'
        p12 = path_dfs + 'period12.csv'
        p13 = path_dfs + 'period13.csv'
        p14 = path_dfs + 'period14.csv'
        p15 = path_dfs + 'period15.csv'
        p16 = path_dfs + 'period16.csv'
        p17 = path_dfs + 'period17.csv'
        p18 = path_dfs + 'period18.csv'
        p19 = path_dfs + 'period19.csv'


        # Then check in which period the date falls, and open right dataframe
        periods = [p0_range, p1_range, p2_range, p3_range, p4_range, p5_range, p6_range, p7_range, p8_range, p9_range, p10_range, p11_range, p12_range, p13_range, p14_range, p15_range, p16_range, p17_range, p18_range, p19_range]
        dfs = [p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19]

        try:
            for i, period in enumerate(periods):
                if dt in period:
                    pol_actors = pd.read_csv(dfs[i])

                    # I delete trailing whitespaces
                    pol_actors['full_name'] = pol_actors['full_name'].str.strip()
                    pol_actors['first_name'] = pol_actors['first_name'].str.strip()
                    pol_actors['last_name'] = pol_actors['last_name'].str.strip()
                    break
            return pol_actors
        except UnboundLocalError:
            # I add this clause in case there are posts that do not fall within the borders of the periods.
            print("this date falls out of the current scope (30/6/2009 - 22/6/2022). The date entered was: " + date + 
                    ". Therefore, we will open the first period, but you should remove this line from the dataset or expand the periods.")
            pol_actors = pd.read_csv(dfs[0])
            return pol_actors

def return_database():
        d_sample['pol_actors'] = d_sample['Post Created Date'].apply(select_database)
        
        return d_sample


def get_info(series):
        """ 
        Helper function that retrieves the information from the database
        """
        infos = {}
        # Name
        infos['name'] = series.full_name.values.tolist()

        # Party
        infos['party'] = series.party.values.tolist()

        # Position: any row where the value is not zero
        srs_dict = dict(list(series.groupby(series.index)))
        for k, v in srs_dict.items():
            check = v.columns[(v==1).any()]
            if len(check) > 0:
                infos['position'] = check.to_list()
            else:
                infos['position'] = []
        
        return infos

def collect_info_subjects(row): # subjects, pol_acts
        """ 
        A helper function that gathers all relevant information for each subject.
        """
        subjects = row['ners']
        pol_acts = row['pol_actors']

        info_subjects = []
        for subject in subjects:
            if subject['label'] == 'PERSON':

                last_name_pattern = re.escape(str(subject['entity']))
                if any(re.search(last_name_pattern, name) for name in pol_acts['last_name']):
                    info_subjects.append(get_info(pol_acts[pol_acts['last_name'].str.contains(last_name_pattern)]))
                    continue
            
                full_name_pattern = re.escape(str(subject['entity']))
                if any(re.search(full_name_pattern, name) for name in pol_acts['full_name']):
                    info_subjects.append(get_info(pol_acts[pol_acts['full_name'].str.contains(full_name_pattern)]))
                    continue
            
                first_name_pattern = re.escape(str(subject['entity']))
                if any(re.search(first_name_pattern, name) for name in pol_acts['first_name']):
                    info_subjects.append(get_info(pol_acts[pol_acts['first_name'].str.contains(first_name_pattern)]))
                    continue
        return info_subjects

def collect_info_subjects_party(row): # subjects, pol_acts
        """ 
        A helper function that collects the parties mentioned in the text.
        """
        subjects = row['ners']
        pol_acts = row['pol_actors']

        info_subjects_party = []
        for subject in subjects:
           if subject['label'] == 'ORG':
                pattern = re.escape(str(subject['entity']))
                if any(re.search(pattern, party, flags=re.IGNORECASE) for party in pol_acts['party']):
                    info_subjects_party.append(str(subject['entity']))
        return info_subjects_party


def collect_info_maker(row): #maker, pol_acts
        """ 
        A helper function that gathers the information of the maker.
        """
        maker = row['Page Name']
        
        pol_acts = row['pol_actors']
        info_maker = get_info(pol_acts[pol_acts['full_name'] == str(maker).strip()])
        return info_maker
    
def info_to_data():
        """ 
        A function that saves the relevant information of the maker and subjects in the dataframe.
        """
        d_sample['info_subject'] = d_sample.apply(collect_info_subjects, axis=1)
        d_sample['info_subject_party'] = d_sample.apply(collect_info_subjects_party, axis=1)
        d_sample['info_maker'] = d_sample.apply(collect_info_maker, axis=1)

        return d_sample


def coalition(df):
        """ 
        A helper function that returns which parties were/ are in the coalition on federal & Flemish level per period, based on the pol-actors file.
        It takes the previously made database that is stored as a dataframe within the main dataframe as input.
        """

        parties_vl = []
        parties_be = []

        party_vl = df[df['coalition_vl'] ==1]['party']
        party_be = df[df['coalition_be'] ==1]['party']
        for i in party_vl:
            parties_vl.append(i)
        for i in party_be:
            parties_be.append(i)
        
        parties_vl = set(parties_vl)
        parties_be = set(parties_be)
        
        return parties_vl, parties_be
    
def return_coalition():
        d_sample['coalition'] = d_sample['pol_actors'].apply(coalition)
        return d_sample


def decision(series):
        answer = "placeholder"
        vl_coal = [i for i in series['coalition'][0]]
        be_coal = [i for i in series['coalition'][1]]

        try:
            series['info_maker']['party'][0]
        except IndexError:
            answer = "Error retrieving info maker: this maker did not have a public function at the time the post was made."
            return answer

        # Specific references either general, federal, or flemish:
        match_general = re.search(r'([Mm]inister|meerderheidspartijen)', series['Message_x'])
        match_vl = re.search(r'([Mm]inister-president|([Dd]e) [V]vlaamse regering\b)', series['Message_x'])
        match_be = re.search(r'(Vivaldi|vivaldicoalitie|vivaldiregering|[Pp]aars-[Gg]roen|paarsgroen|[Zz]weedse coalitie|kamikazecoalitie|tripartite|([Dd]e) \bregering\b|[Dd]e \boverheid\b|[Dd]e \bstaat\b|staatssecretaris|[Pp]remier|[Ee]rste minister|federale regering\b|[Dd]e huidige regering\b|[Dd]eze regering\b)', series['Message_x'])

        # Name of federal government:
        mask = series.pol_actors['prime_minister_be'] == 1
        pm_name = series.pol_actors.loc[mask, 'last_name'].iloc[0] # I take the surname of the person that has the function prime minister in that period
        name_gov = r'[Rr]egering-'+pm_name
        match_gov_be = re.search(name_gov, series['Message_x']) 

        # Name of flemish government:
        mask2 = series.pol_actors['prime_minister_vl'] == 1
        pm_name2 = series.pol_actors.loc[mask2, 'last_name'].iloc[0] # I take the surname of the person that has the function prime minister in that period
        name_gov_vl = r'[Rr]egering-'+pm_name2
        match_gov_vl = re.search(name_gov_vl, series['Message_x']) 

        # Define subject and subject party
        try:
            subject = series['info_subject'][0]
        except IndexError:
            subject = series['info_subject']
        try:
            subject_party = series['info_subject_party'][0]
        except IndexError:
            subject_party = series['info_subject_party']


        # If there is no reference whatsoever: no subject
        if (not match_general and not match_be and not match_vl and not match_gov_be and not match_gov_vl and subject in [[], {'name': [], 'party': []}, {'name': [], 'party': [], 'position': []}] and subject_party in [[], {'name': [], 'party': []}, {'name': [], 'party': [], 'position': []}]):
            answer = "no subject"

        else:

            # I work with a variable here, to make sure that no lines go lost if only one condition is False
            flag = False
            if subject:
                if subject not in [[], {'name': [], 'party': []}, {'name': [], 'party': [], 'position': []}]:
                    if series['info_maker']['party'][0] == subject['party'][0]:

                        answer = "same party"
                        flag = True
                    elif (subject['party'][0] in ['Vooruit', 'sp.a']) and (series['info_maker']['party'][0] in ['Vooruit', 'sp.a']):

                        answer = "same party"
                        flag = True
                    else:
                        pass
                else:
                    pass
            elif subject_party:
                # I add this line here because there are some lines that have a match but empty references to subject/ party
                if subject_party not in [[], {'name': [], 'party': []}, {'name': [], 'party': [], 'position': []}]:
                    if (series['info_maker']['party'][0] == subject_party):# or (series['info_maker']['party'][0] == subject['party'][0]):
                        answer = "same party"
                        flag = True
                    elif (subject_party in ['Vooruit', 'sp.a']) and (series['info_maker']['party'][0] in ['Vooruit', 'sp.a']):
                        answer = "same party"
                        flag = True
                    else:
                        pass
                else:
                    pass
        
            if not flag:

                ref_subject = False
                if subject:
                
                    if subject not in [[], {'name': [], 'party': []}, {'name': [], 'party': [], 'position': []}]:
                        ref_subject = True
                        if 'parl_EU' in subject['position']:
                            answer = "Outside Belgium"
                            
                            # If this is correct, we want to exit the loop
                            return answer
                    
                        elif 'gov_be' in subject['position']:
                        
                            # Explicit references to a politician:
                            if series['info_maker']['party'][0] in be_coal:
                                answer = "same federal coalition"
                        
            
                            else: 
                                answer = "not same party/ coalition"


                        elif 'gov_vl' in subject['position']:
                        
                            if series['info_maker']['party'][0] in vl_coal:
                                answer = "same flemish coalition"
                        
                            else:
                                answer = "not same party/ coalition"

                        else: 
                            answer = "not same party/ coalition"
                
                    else:
                        pass

                ref_party = False
                if not ref_subject:
                
                    if subject_party:
                        if subject_party not in [[], {'name': [], 'party': []}, {'name': [], 'party': [], 'position': []}]:
                            ref_party = True
                    
                            if subject_party in be_coal:
                                if series['info_maker']['party'][0] in be_coal:
                                    answer = "same federal coalition"
  
                                else:
                                    answer = "not same party/ coalition"

                            
                            elif subject_party in vl_coal:
                                if series['info_maker']['party'][0] in vl_coal:
                                    answer = "same flemish coalition"
                    
                                else: 
                                    answer = "not same party/ coalition"
                            
                    
                            else:
                                answer = "not same party/ coalition"

                        else:
                            pass

                # If there is no subject, there can still be a match. If there is, and the maker is in the same coalition: option b
                if not ref_party:
                            if (match_general or match_be or match_gov_be) and (series['info_maker']['party'][0] in be_coal):
        
                                answer = "same federal coalition"
                            elif (match_general or match_vl or match_gov_vl) and (series['info_maker']['party'][0] in vl_coal):
                                answer = "same flemish coalition"
                            else:

                                answer = "not same party/ coalition"

        return answer
    

def return_decision(d_sample):
        l = []
        for i in range(len(d_sample)):
            d = decision(d_sample.iloc[i])
            l.append(d)
        
        d_sample['final_decision'] = l

        # To have a cleaner output, we drop the dfs as parts of each row
        d_sample = d_sample.drop('pol_actors', axis=1)

        return d_sample

# There are a bunch of misspelled names/ names with additions, so replace them with the name as in the file
d_sample.loc[d_sample['Page Name'] == 'Emmily Talpe - Burgemeester Ieper', 'Page Name'] = 'Emmily Talpe'
d_sample.loc[d_sample['Page Name'] == 'Johan Verstreken Vriendenpagina', 'Page Name'] = 'Johan Verstreken'
d_sample.loc[d_sample['Page Name'] == 'Michael Freilich - Kamerlid', 'Page Name'] = 'Michael Freilich'
d_sample.loc[d_sample['Page Name'] == 'Lanjri Nahima', 'Page Name'] = 'Nahima Lanjri'
d_sample.loc[d_sample['Page Name'] == 'Anke Van Dermeersch', 'Page Name'] = 'Anke Van dermeersch'
d_sample.loc[d_sample['Page Name'] == "Christoph D'Haese-pagina", 'Page Name'] = "Christoph D'Haese"
d_sample.loc[d_sample['Page Name'] == 'Cindy Franssen.', 'Page Name'] = 'Cindy Franssen'
d_sample.loc[d_sample['Page Name'] == 'Katleen Bury Federaal Volksvertegenwoordiger', 'Page Name'] = 'Katleen Bury'
d_sample.loc[d_sample['Page Name'] == 'Stefaan De Clerck 2018', 'Page Name'] = 'Stefaan De Clerck'
d_sample.loc[d_sample['Page Name'] == 'Sam Van Rooy', 'Page Name'] = 'Sam van Rooy'
d_sample.loc[d_sample['Page Name'] == 'Jean Marie Dedecker', 'Page Name'] = 'Jean-Marie Dedecker'
d_sample.loc[d_sample['Page Name'] == 'Ben Segers - Antwerpen', 'Page Name'] = 'Ben Segers'
d_sample.loc[d_sample['Page Name'] == 'Volg Marino Keulen op de voet', 'Page Name'] = 'Marino Keulen'
d_sample.loc[d_sample['Page Name'] == 'Reccino Van Lommel - Vlaams Belang', 'Page Name'] = 'Reccino Van Lommel'
d_sample.loc[d_sample['Page Name'] == 'Erik Gilissen - Vlaams Belang', 'Page Name'] = 'Erik Gilissen'
d_sample.loc[d_sample['Page Name'] == 'Celia Groothedde Ledoux', 'Page Name'] = 'Celia Groothedde'


# Apply
print('Applying NER...')
apply_ner()
print('NER applied.')

print('Checking database...')
return_database()
print('Info databased accessed.')

print('Extract info...')
info_to_data()
print('Info extracted.')

print('Extracting coalitions...')
return_coalition()
print('Coalitions extracted.')

print('Making decision subject...')
output = return_decision(d_sample)
print('Decision subject made.')
print(f'output subject value count: {output["final_decision"].value_counts()}')

output = output[['ID', 'ID_unit', 'Message_x', 'Message_y', 'Sentences',
       'Page Name', 'info_maker', 'Post Created Date', 'group', 'preds_representation',
       'preds_binary','final_decision' ]]
output.rename(columns= {'final_decision': 'preds_subject'}, inplace=True)

output.head()


# Join with non-claims
print('Joining with non-claims...')

non_claims = d[d['preds_binary'] == 0]
# Contrary to training data, the creation date is in a different format, so adapt
non_claims['Post Created Date'] = non_claims['Post Created Date'].apply(str)
non_claims['Post Created Date'] = non_claims['Post Created Date'].str.split(' ')
non_claims['Post Created Date'] = non_claims['Post Created Date'].apply(lambda x: x[0])

non_claims['preds_subject'] = 0
data = pd.concat([output, non_claims])
print('Joined with non-claims.')
print(len(data), len(d))

print(f'output subject value count: {data["preds_subject"].value_counts()}')

print('Saving output...')
data.drop('Unnamed: 0', axis=1, inplace=True)
data.to_excel('data/final_data_subject.xlsx')
print('Output saved.')