Notes

from re import L
from bs4 import BeautifulSoup
from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver
import time
import json
import csv
from random import randint
from html.parser import HTMLParser
USER_AGENT = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'}
SEARCHING_URL = "http://www.bing.com/search?q="
COUNT_QUERY = "&count=30"
options = webdriver.ChromeOptions()
options.add_argument('--ignore-certificate-errors')
options.add_argument('--incognito')
options.add_argument('--headless')
driver = webdriver.Chrome(ChromeDriverManager().install(), chrome_options=options)

def search(query, sleep=True):
if sleep: # Prevents loading too many pages too soon
t = randint(5, 20)
print(f'nsleep for {t}s')
time.sleep(t)

temp_url = '+'.join(query.split()) #for adding + between words for the query
url = SEARCHING_URL + temp_url + COUNT_QUERY
driver.get(url)
soup = BeautifulSoup(driver.page_source, "html.parser")
# soup = BeautifulSoup(requests.get(url, headers=USER_AGENT).text, "html.parser")
print(f'URL: {url}nQuery: {query}')
new_results = scrape_search_result(soup)
return new_results

def scrape_search_result(soup):
raw_results = soup.find_all("li", attrs={"class": "b_algo"})
results = []
#implement a check to get only 10 results and also check that URLs must not be duplicated
links_set = set()
for result in raw_results:
link = result.find('a').get('href')
if link == None: break
cleaned = clean_link(link)
if cleaned in links_set: continue
cont = False
for ng in ['youtube', 'bing', 'microsoft']:
if ng in cleaned[:10]:
cont = True
break
if cont: continue
results.append(link)
links_set.add(cleaned)
if len(results) == 10:
break
print(f'Links - {len(results)}: n{results}')
return results

def clean_link(link):
link = link.replace("https://", "")
link = link.replace("http://", "")
link = link.replace("www.", "")
link = link[:-1] if link[-1] == "/" else link
return link

# --- import queries ---
queries = []
with open('100QueriesSet1.txt') as queries_file:
for q in queries_file:
queries.append(q.rstrip())
queries = queries[25:40]
# 25 index: 'The precipitaion of a cougar' left
# --- End import queries ---

# # ---Scarpe and store---
scraping_result = {}
for query in queries:
scraping_result[query] = search(query, True)
if scraping_result[query] == []: break
driver.close()
driver.quit()
print(scraping_result)
with open("data.json", "w") as outfile:
json.dump(scraping_result, outfile, indent=4)
# # ---Scrape ends ----

# --- import dummy data ----
# scraping_result = {}
# with open('First5.json') as dummy:
# scraping_result = json.load(dummy)
# --- END import dummy data ----

# ---Google results ---
# google_res = {}
# with open('google_test.json') as g_res:
# google_res = json.load(g_res)
# --- End ---

# total_overlap, total_percent, total_rho = 0, 0, 0
# stats = []
# for idx, query in enumerate(queries):
# bing = scraping_result[query]
# google = google_res[query]

# map_bing = {}
# for i, link in enumerate(bing):
# link = clean_link(link)
# map_bing[link] = i

# overlaps, sum = 0, 0
# overlap_match = False
# for i, link in enumerate(google):
# link = clean_link(link)
# if link in map_bing:
# overlaps += 1
# sum += ((i - map_bing[link]) ** 2)
# if i == map_bing[link]:
# overlap_match = True
# else:
# overlap_match = False
# if overlaps == 0:
# rho = 0
# elif overlaps == 1:
# rho = 1 if overlap_match else 0
# else:
# rho = 1 - ((6 * sum) / (overlaps * (overlaps ** 2 - 1)))

# stats.append(['Query ' + str(idx+1), overlaps, overlaps*100/10, rho])
# total_overlap += overlaps
# total_rho += rho
# total_percent += overlaps*100/10

# with open("hw1.csv", "w", newline='') as file:
# writer = csv.writer(file)
# writer.writerow(["Queries", "No of Overlapping Results", "Percent Overlap", "Spearman Coefficient"])
# writer.writerows(stats)
# writer.writerow(["Averages", total_overlap/len(queries), total_percent/len(queries), total_rho/len(queries)])