pilabsDefinitionExtraction/Process.py at main · gkiler/pilabsDefinitionExtraction · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
from asyncio import gather
# from utils.proceed import proceed
# from utils.information import *
import pandas as pd
import time
from bs4 import BeautifulSoup, SoupStrainer
import io
from pathvalidate import sanitize_filename
from tqdm import tqdm
tqdm.pandas()

CSV_SOURCE="C:\\Users\\gwenk\\OneDrive\\Documents\\GitHub\\pilabsDefinitionExtraction\\data.csv"

def gather_definitions():
    # do stuff to gather definitions
    print("> Gathering definitions...")

    df_saved = pd.read_csv(CSV_SOURCE, nrows=None)
    print(df_saved.head(3))

    beg_t = time.time()

    def parseDefinitions(raw_html, source_name, concept_name, source_url, date_time_scraped, concept_type, CUI):
        if "drugs.com" in source_name.lower(): # drugs.com is the source
            parse_only = SoupStrainer(attrs = {"class" : "contentBox"})
            bs = BeautifulSoup(raw_html, "lxml", parse_only=parse_only)
            mydivs = bs.find_all("div", attrs={"class": "contentBox"})
            string = ""
            if len(mydivs) == 0:
                return None
            interest_p = mydivs[0].find_all("p", attrs={"class" : None})[0:]
            string += ' \n SOURCEURLBEGIN ' + source_url + ' SOURCEURLEND\n'
            string += ' \n DATETIMEBEGIN ' + date_time_scraped + ' DATETIMEEND\n'
            string += ' \n SOURCENAMEBEGIN ' + source_name + ' SOURCENAMEEND\n'
            string += ' \n CONCEPTTYPEBEGIN ' + concept_type + ' CONCEPTTYPEEND\n'
            string += ' \n CUIBEGIN ' + str(CUI) + ' CUIEND\n'
            string += ' \n DRUGBEGIN ' + concept_name + ' DRUGEND\n'
            for p in interest_p:
                string += p.text
            with io.open('drug_concat' + '.txt', 'a', encoding='utf-8') as f:
                f.write(string)
        elif "mayoclinic" in source_name.lower(): # Mayoclinic is the source
            parse_only = SoupStrainer(attrs = {"id" : "main-content"})
            bs = BeautifulSoup(raw_html, "lxml", parse_only=parse_only)
            mydivs = bs.find_all("div", attrs={"id" : "main-content"})
            string = ""
            if len(mydivs) == 0:
                return None
            interest_p = mydivs[0].find_all("p", attrs={"class" : None})[0:]
            string += ' \n SOURCEURLBEGIN ' + source_url + ' SOURCEURLEND\n'
            string += ' \n DATETIMEBEGIN ' + date_time_scraped + ' DATETIMEEND\n'
            string += ' \n SOURCENAMEBEGIN ' + source_name + ' SOURCENAMEEND\n'
            string += ' \n CONCEPTTYPEBEGIN ' + concept_type + ' CONCEPTTYPEEND\n'
            string += ' \n CUIBEGIN ' + str(CUI) + ' CUIEND\n'
            string += ' \n DRUGBEGIN ' + concept_name + ' DRUGEND\n'
            for p in interest_p:
                string += p.text
            with io.open('drug_concat' + '.txt', 'a', encoding='utf-8') as f:
                f.write(string)
        elif "webmd" in source_name.lower(): # WebMD is the source
            parse_only = SoupStrainer(attrs = {"class" : "monograph-content monograph-content-holder"})
            bs = BeautifulSoup(raw_html, "lxml", parse_only=parse_only)
            mydivs = bs.find_all("div", attrs={"class" : "monograph-content monograph-content-holder"})
            string = ""
            if len(mydivs) == 0:
                return None
            interest_p = mydivs[0].find_all("p", attrs={"class" : None})[0:]
            string += ' \n SOURCEURLBEGIN ' + source_url + ' SOURCEURLEND\n'
            string += ' \n DATETIMEBEGIN ' + date_time_scraped + ' DATETIMEEND\n'
            string += ' \n SOURCENAMEBEGIN ' + source_name + ' SOURCENAMEEND\n'
            string += ' \n CONCEPTTYPEBEGIN ' + concept_type + ' CONCEPTTYPEEND\n'
            string += ' \n CUIBEGIN ' + str(CUI) + ' CUIEND\n'
            string += ' \n DRUGBEGIN ' + concept_name + ' DRUGEND\n'
            for p in interest_p:
                string += p.text
            with io.open('drug_concat' + '.txt', 'a', encoding='utf-8') as f:
                f.write(string)
        elif "medline" in source_name.lower(): # Medline is the source
            # section-body
            parse_only = SoupStrainer(attrs = {"class" : "section-body"})
            bs = BeautifulSoup(raw_html, "lxml", parse_only=parse_only)
            mydivs = bs.find_all("div", attrs={"class" : "section-body"})
            string = ""
            if len(mydivs) == 0:
                return None
            interest_p = mydivs[0].find_all("p", attrs={"class" : None})[0:]
            string += ' \n SOURCEURLBEGIN ' + source_url + ' SOURCEURLEND\n'
            string += ' \n DATETIMEBEGIN ' + date_time_scraped + ' DATETIMEEND\n'
            string += ' \n SOURCENAMEBEGIN ' + source_name + ' SOURCENAMEEND\n'
            string += ' \n CONCEPTTYPEBEGIN ' + concept_type + ' CONCEPTTYPEEND\n'
            string += ' \n CUIBEGIN ' + str(CUI) + ' CUIEND\n'
            string += ' \n DRUGBEGIN ' + concept_name + ' DRUGEND\n'
            for p in interest_p:
                string += p.text
            with io.open('drug_concat' + '.txt', 'a', encoding='utf-8') as f:
                f.write(string)
        elif "cdc" in source_name.lower():
            # section-body
            parse_only = SoupStrainer(attrs = {"class" : "col-md-12 splash-col"})
            bs = BeautifulSoup(raw_html, "lxml", parse_only=parse_only)
            mydivs = bs.find_all("div", attrs={"class" : "col-md-12 splash-col"})
            string = ""
            if len(mydivs) == 0:
                return None
            interest_p = mydivs[0].find_all("p", attrs={"class" : None})[0:]
            string += ' \n SOURCEURLBEGIN ' + source_url + ' SOURCEURLEND\n'
            string += ' \n DATETIMEBEGIN ' + date_time_scraped + ' DATETIMEEND\n'
            string += ' \n SOURCENAMEBEGIN ' + source_name + ' SOURCENAMEEND\n'
            string += ' \n CONCEPTTYPEBEGIN ' + concept_type + ' CONCEPTTYPEEND\n'
            string += ' \n CUIBEGIN ' + str(CUI) + ' CUIEND\n'
            string += ' \n DRUGBEGIN ' + concept_name + ' DRUGEND\n'
            for p in interest_p:
                string += p.text
            with io.open('drug_concat' + '.txt', 'a', encoding='utf-8') as f:
                f.write(string)
        elif "nhs" in source_name.lower():
            # section-body
            parse_only = SoupStrainer(attrs = {"class" : "js-guide cf guide"})
            bs = BeautifulSoup(raw_html, "lxml", parse_only=parse_only)
            mydivs = bs.find_all("div", attrs={"class" : "tab js-guide__section guide__section active"})
            string = ""
            if len(mydivs) == 0:
                return None
            interest_p = mydivs[0].find_all("p", attrs={"class" : None})[0:]
            string += ' \n SOURCEURLBEGIN ' + source_url + ' SOURCEURLEND\n'
            string += ' \n DATETIMEBEGIN ' + date_time_scraped + ' DATETIMEEND\n'
            string += ' \n SOURCENAMEBEGIN ' + source_name + ' SOURCENAMEEND\n'
            string += ' \n CONCEPTTYPEBEGIN ' + concept_type + ' CONCEPTTYPEEND\n'
            string += ' \n CUIBEGIN ' + str(CUI) + ' CUIEND\n'
            string += ' \n DRUGBEGIN ' + concept_name + ' DRUGEND\n'
            for p in interest_p:
                string += p.text
            with io.open('drug_concat' + '.txt', 'a', encoding='utf-8') as f:
                f.write(string)
        else:
            return None
        # else:
        #     return None

    print("> Extracting many definitions (this may take a while)...")
    with io.open('drug_concat.txt','w',encoding='utf-8') as f:
        f.write('')
    df_saved["definition"] = df_saved.progress_apply(lambda x : parseDefinitions(x["raw_html"], x["source_name"], x["name"], x["source_url"], x["date_time_scraped"], x["concept_type"], x["CUI"]), axis=1)
    # parseDefinitions(x["raw_html"], x["source_name"], x["name"])
    end_t = time.time()
    print(f"> Total time: {end_t - beg_t}s")

    # beginning = time.time()
    # print(f"> Saving...")
    # df_saved.reset_index()
    # # df_saved.to_csv(DATA_PATH + "/" + OUT_FILE, index=False)
    # final = time.time()
    # print(f"> Done saving.")
    # print(f"> Time taken: {final - beginning}s")

gather_definitions()