Web Scraping from Ynet with Python¶
In [ ]:
#installing pandas libraries
!pip install pandas-datareader
!pip install --upgrade html5lib==1.0b8
#There is a bug in the latest version of html5lib so install an earlier version
#Restart kernel after installing html5lib
In [3]:
#import libraries
import requests
from bs4 import BeautifulSoup,Comment
from lxml.etree import fromstring
import urllib.request
import json
import pandas as pd #pandas library
from pandas_datareader import data #data readers (google, html, etc.)
#The following line ensures that graphs are rendered in the notebook (and not in another window)
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt #Plotting library
import datetime as dt #datetime for timeseries support
In [230]:
#Get article links from YNET and return in list
def get_article_links(home_page_url):
url_site="https://www.ynet.co.il/"
url=[]
response = requests.get(home_page_url)
soup = BeautifulSoup(response.content, 'html.parser')
article_links = soup.find_all('a', class_="mta_title")
for link in article_links:
if 'http' not in link.get('href'):
url.append(url_site+link.get('href'))
elif 'frogi' not in link.get('href') : #exclde other sites
url.append(link.get('href'))
return url
get_article_links("https://www.ynet.co.il/")
Out[230]:
In [244]:
#Get article tags from YNET and return it in list
def get_article_tags(url):
tags=[]
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
url_tag = soup.find_all('div', class_="tagName")
url_tag2=soup.find_all('a', class_="articletags_link")
#if no tags return null value
if len(url_tag) == 0 and len(url_tag2) == 0:
return 'null'
try:
if len(url_tag) == 0:
for tag in url_tag2:
tag=tag['href'].rsplit('/', 1)[-1]
tags.append(tag)
return tags
elif len(url_tag) != 0:
for tag in url_tag:
tags.append(tag.get_text())
return tags
else:
return 'null'
except:
return 'null'
get_article_tags("https://www.ynet.co.il//judaism/article/HJAbsxaZP")
Out[244]:
In [232]:
#Q3 - Get article_talkbacks from YNET using api and return list of dictionarys
def get_article_talkbacks(url):
#Find the ID of the URL
right_url=url.rsplit('/', 1)[-1].strip('#autoplay')
#if No talkback return zero object
zero= [{'Body': '','Like': 0,'Dislike': 0,'ID': right_url }]
#Exclude URL Id's who have dots in it(.)
if '.' not in right_url:
fix_url="https://www.ynet.co.il/iphone/json/api/talkbacks/list/%s/end_to_start/1" % (right_url)
response = requests.get(fix_url).json()
dic=dict()
body=list()
lis = list()
try:
for i in response ["rss"]["channel"]["item"]:
dic["Body"]=i['text']
dic["Like"]=i['likes']
dic["Dislike"]=i['unlikes']
dic["ID"]=right_url
dictionary_copy = dic.copy()
lis.append(dictionary_copy)
#body.append(i['text'])
df=pd.DataFrame.from_dict(lis)
return lis
except:
return zero
else:
return zero
h=get_article_talkbacks('https://www.ynet.co.il//articles/0,7340,L-5767587,00.html')
h
Out[232]:
In [233]:
#get article published date for Q4a
def get_article_date(url):
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
url_date = soup.find('div', class_="date")
try:
return url_date.get_text()
except:
return current_time()
get_article_date("https://www.ynet.co.il/article/H1lngWDWP#autoplay ")
Out[233]:
In [234]:
#Get current time for the Crawler for Q4a
import datetime #import datetime libray
#get the current datetime
def current_time():
now = datetime.datetime.now()
return now
In [235]:
#Get article header for Q4a
def get_article_header(url):
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
url_header = soup.find('h1', class_="mainTitle")
try:
return url_header.get_text()
except:
try:
return soup.find('h1', class_="art_header_title").get_text()
except:
try:
return soup.find('h1', class_="ahxnet_title cp-color cp-big-h1").get_text()
except:
return 'error'
get_article_header("https://www.ynet.co.il//judaism/article/HJAbsxaZP")
Out[235]:
In [219]:
# Using realated functions in order to create
#panda dataframe named articles and panda dataframe named talkbacks
article_links=get_article_links("https://www.ynet.co.il/")
new_tags=[]
new_comment=[]
article_date=[]
time=[]
article_header=[]
article_comment=[]
article_url=[]
for article in article_links:
new_tags.append(get_article_tags(article))
article_date.append(get_article_date(article))
time.append(current_time())
article_header.append(get_article_header(article))
article_comment=article_comment+(get_article_talkbacks(article))
article_url.append(article.rsplit('/', 1)[-1].strip('#autoplay'))
#pd dataframe named articles
articles = pd.DataFrame ( {'tags': new_tags, 'enter time': time,
'article title': article_header,'article_date':article_date,'ID':article_url} )
articles
Out[219]:
In [220]:
#create pd dataframe named talkbacks
talkbacks=pd.DataFrame(article_comment)
talkbacks
Out[220]:
In [236]:
# for each report Calculate total comments,LIkes and Dislikes
#total comments
total_comments=talkbacks.groupby('ID')['Body'].count()
#total likes
total_likes=talkbacks.groupby('ID')['Like'].sum()
#total dislikes
total_dislikes=talkbacks.groupby('ID')['Dislike'].sum()
print(total_comments)
print(total_dislikes)
print(total_likes)
In [239]:
#find most frequent tag
#Transform each tag of a list-like to a row
articles_tags=articles.explode('tags')
#To Pretiffy the data we remove null values
articles_tags=articles_tags[articles_tags.tags != 'null']
#Find top 10 frequent tag
common_tags=articles_tags.groupby('tags')['tags'].count().sort_values(ascending=False).head(10)
print(common_tags)
common_tags.plot.bar()
plt.show()
In [241]:
#Find most frequent talkbacks -The top 10
#Merge talkbacks and articles pandas df with left join operation
df_merge= pd.merge(talkbacks, articles, on='ID', how='left')
df_merge
#Show most frequent talkbacks with the title-The top 10
max_comment=df_merge.groupby('article title')['Body'].count().sort_values(ascending=False)
max_comment.head(10)
Out[241]:
In [242]:
# Present above section in bar chart
max_comment.head(10).plot.bar()
plt.xlabel('header')
plt.xticks(rotation=90)
plt.show()
In [243]:
#Save dataframes as CSV file
articles.to_csv(r'C:\Users\Alon2\Documents\Python\final_assighment\articles.csv', encoding='utf-8-sig' )
talkbacks.to_csv(r'C:\Users\Alon2\Documents\Python\final_assighment\talkbacks.csv', encoding='utf-8-sig' )