Notes

import pandas as pd
import glob
import requests
from bs4 import BeautifulSoup
import requests
import webbrowser
from user_agent import generate_user_agent, generate_navigator
import urllib
import re
from datetime import datetime
startTime = datetime.now()
all_data = []
d = 0

def scrapeFromGoogle(query,Ref_ID,Reference):
queryData = query.strip()
diffparam = re.findall('(. d{4})',queryData)
if diffparam:
if queryData.split(diffparam[-1]):
queryData = queryData.split(diffparam[-1])[0].strip()
querydata = urllib.parse.quote_plus(queryData)
headers = {'User-Agent': generate_user_agent()}
url = f"https://pubmed.ncbi.nlm.nih.gov/?term={querydata}"
resp = requests.get(url, headers=headers)
DataLWithAffiliations = []
results = []
if resp.status_code == 200:
soup = BeautifulSoup(resp.content, "html.parser")
if not soup.findAll(attrs={'class' : 'heading-title'}):
url = f"https://pubmed.ncbi.nlm.nih.gov/?term={querydata}"
resp = requests.get(url, headers=headers)
soup = BeautifulSoup(resp.content, "html.parser")
if not soup.findAll(attrs={'class' : 'heading-title'}):
url = f"https://pubmed.ncbi.nlm.nih.gov/?term={querydata}"
resp = requests.get(url, headers=headers)
soup = BeautifulSoup(resp.content, "html.parser")
PMIDLink = ''
if soup.find(attrs={'class' : 'id-link'}):
PMIDLink = soup.find(attrs={'class' : 'id-link'}).get("href")
if soup.findAll(attrs={'class' : 'authors-list-item'}):
for nameData in soup.findAll(attrs={'class' : 'authors-list-item'}):
nameData_Id = nameData.find(attrs={'class' : 'affiliation-link'})
Full_Name = nameData.find(attrs={'class' : 'full-name'})
result = {}
result['Name'] =''
if nameData_Id:
result['nameData_Id'] = nameData_Id.text.strip()
else:
result['nameData_Id'] = ''
result['Full_Name']=Full_Name.text.strip()
result['Link_I'] = str(resp.url)
result['Reference'] = str(Reference).strip()
result['Ref_ID'] = str(Ref_ID)
result['DOI'] = str(PMIDLink)
results.append(result)
results = [dict(y) for y in set(tuple(x.items()) for x in results)]
else:
results.append({'Link_I':str(resp.url),'DOI':str(PMIDLink),'Full_Name':'','Ref_ID': str(Ref_ID),'Reference':str(Reference).strip()})
if soup.find(attrs={'class' : 'extended-article-details'}):
if soup.find(attrs={'class' : 'extended-article-details'}).text.strip():
for affiliations in soup.find(attrs={'class' : 'extended-article-details'}).select("li"):
if affiliations.select("sup"):
for resultdata in results:
if affiliations.select("sup")[0].text.strip() == resultdata['nameData_Id']:
resultdata['Affiliation'] = affiliations.text.split(affiliations.select('sup')[0].text)[1].strip()
DataLWithAffiliations.append(resultdata)
else:
for resultdata in results:
resultdata['Affiliation'] = ''
DataLWithAffiliations.append(resultdata)
else:
for resultdata in results:
resultdata['Affiliation'] = ''
DataLWithAffiliations.append(resultdata)
else:
for resultdata in results:
resultdata['Affiliation'] = ''
DataLWithAffiliations.append(resultdata)
print(results)
results = [dict(y) for y in set(tuple(x.items()) for x in results)]
return results

for f in glob.glob(r"C:Usersbarada.jenaDesktopScynexis7_APRWeb_scraping_pull_v0_1.csv"):
all_data.append(pd.read_csv(f,na_filter= False,engine='python'))

df = pd.concat(all_data, ignore_index=True)
allJsonData = df.to_dict(orient='records')

AllListDicts = []
for item in allJsonData:
Names = []
if item:
# Calling the function with Input parameters # https://pubmed.ncbi.nlm.nih.gov/
data = scrapeFromGoogle(item['Reference'],item['Ref_ID'],item['Reference'])
AllListDicts.extend(data)
if AllListDicts:
df = pd.DataFrame(AllListDicts)
df.to_csv(r'C:Usersbarada.jenaDesktopScynexis7_APRInfluencemappingOutput_%s.csv'%str(datetime.now().strftime("%m-%d-%Y_%H-%M")))

print("Total et al. results are :"+str(d))
print("Script Execution TIME : ",datetime.now() - startTime)