
import pandas as pd
from time import sleep
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from IPython import display


url = 'https://www.sciencefocus.com/nature/why-do-we-find-puppies-so-cute/'


driver = webdriver.Chrome()
driver.get(url)
sleep(5)
html_data = driver.page_source


soup = BeautifulSoup(html_data, 'html.parser')


type(soup.find_all('h1'))

bs4.element.ResultSet


title = soup.find_all('h1')[0].text
title

'Why do we find puppies so cute?'


author = soup.find_all('a', {'class': 'link link--styled'})[0].text
author

'Christian Jarrett'


for div in soup.find_all('div', {'class': 'post__content'}):
	article_text = div.text


article_text

'Asked by: Arnold Hopper, SwanseaAdvertisementWith their big, round eyes, button noses and large heads, puppies share many of the same physical characteristics as human babies. And like babies, as well as kittens, teddies and many cartoon characters, puppies provoke in us an automatic ‘cute response’. They grab our attention, we enjoy looking at them and, at a neural level, they trigger activity associated with reward and also compassion and empathy.In humans, and other animals, this response is an evolved, innate behaviour that motivates adults to look after helpless infants, and to be more sensitive to their needs and feelings. It makes sense, then, that a recent study found that puppies reach peak cuteness at eight weeks of age, just the time when their canine mothers leave them to fend for themselves.Read more:\nDo babies have nightmares?\nIs stroking good for pets?\nAdvertisementSubscribe to BBC Focus magazine for fascinating new Q&As every month and follow @sciencefocusQA on Twitter for your daily dose of fun facts.'


article_text = article_text.replace('\n', ' ')


article_text

'Asked by: Arnold Hopper, SwanseaAdvertisementWith their big, round eyes, button noses and large heads, puppies share many of the same physical characteristics as human babies. And like babies, as well as kittens, teddies and many cartoon characters, puppies provoke in us an automatic ‘cute response’. They grab our attention, we enjoy looking at them and, at a neural level, they trigger activity associated with reward and also compassion and empathy.In humans, and other animals, this response is an evolved, innate behaviour that motivates adults to look after helpless infants, and to be more sensitive to their needs and feelings. It makes sense, then, that a recent study found that puppies reach peak cuteness at eight weeks of age, just the time when their canine mothers leave them to fend for themselves.Read more: Do babies have nightmares? Is stroking good for pets? AdvertisementSubscribe to BBC Focus magazine for fascinating new Q&As every month and follow @sciencefocusQA on Twitter for your daily dose of fun facts.'


url = 'https://google.com'
driver = webdriver.Chrome()
driver.get(url)


search_field = driver.find_element(By.NAME, 'q')


search_field.send_keys('Why are dogs so cute?')
search_field.send_keys(Keys.ENTER)


html_data = driver.page_source


soup = BeautifulSoup(html_data, 'html.parser')


[title.text for title in soup.find_all('h3')]

['The Scientific Reason Behind Why We Find Dogs So Cute',
 'Do dogs know we think they are cute? - The Healthy Journal',
 'Curious Kids: Why is our dog so cute? - The Conversation',
 'Why Do Dogs Love Us? Science Explains - Inverse',
 'Dogs Mentally Understand How Much We Love Them, Canine Cognition ...',
 'Curious Kids: Why is our dog so cute? - The Conversation',
 'The Science of Cute: How Cuteness Makes Us Love Our Dogs',
 "Why Are Dogs So Cute? - Dogs 'N Stuff",
 'Why is my dog so cute? | Ohio Wesleyan University',
 'Science Explains What Makes Your Dog So Cute - WSJ',
 'Why do we find puppies so cute? - BBC Science Focus',
 'What makes a dog cute to a human being? - Quora',
 'Why Are Puppies So Cute? | Psychology Today',
 "What Your Dog's Cute Behaviors Really Mean",
 'More results',
 'Try again']


titles = [title.text for title in soup.find_all('h3')][:14]


titles

['The Scientific Reason Behind Why We Find Dogs So Cute',
 'Do dogs know we think they are cute? - The Healthy Journal',
 'Curious Kids: Why is our dog so cute? - The Conversation',
 'Why Do Dogs Love Us? Science Explains - Inverse',
 'Dogs Mentally Understand How Much We Love Them, Canine Cognition ...',
 'Curious Kids: Why is our dog so cute? - The Conversation',
 'The Science of Cute: How Cuteness Makes Us Love Our Dogs',
 "Why Are Dogs So Cute? - Dogs 'N Stuff",
 'Why is my dog so cute? | Ohio Wesleyan University',
 'Science Explains What Makes Your Dog So Cute - WSJ',
 'Why do we find puppies so cute? - BBC Science Focus',
 'What makes a dog cute to a human being? - Quora',
 'Why Are Puppies So Cute? | Psychology Today',
 "What Your Dog's Cute Behaviors Really Mean"]


del titles[1:5]


titles

['The Scientific Reason Behind Why We Find Dogs So Cute',
 'Curious Kids: Why is our dog so cute? - The Conversation',
 'The Science of Cute: How Cuteness Makes Us Love Our Dogs',
 "Why Are Dogs So Cute? - Dogs 'N Stuff",
 'Why is my dog so cute? | Ohio Wesleyan University',
 'Science Explains What Makes Your Dog So Cute - WSJ',
 'Why do we find puppies so cute? - BBC Science Focus',
 'What makes a dog cute to a human being? - Quora',
 'Why Are Puppies So Cute? | Psychology Today',
 "What Your Dog's Cute Behaviors Really Mean"]


articles = []


len(soup.find_all('div', {'class': 'xpdopen'}))

1


articles.append(soup.find_all('div', {'class': 'xpdopen'})[0].find_all('a')[0]['href'])

soup.find_all('div', {'class': 'xpdopen'})[0].find_all('a')[0]['href']

'https://www.dabl.com/dabl-home/dabl-home/articles/scientific-reason-behind-why-we-find-dogs-so-cute#:~:text=In%20the%20past%2C%20research%20suggested,dogs%20are%20a%20perfect%20example.'


len(soup.find_all('div', {'data-header-feature': '0'}))

9


soup.find_all('div', {'data-header-feature': '0'})[0].find_all('a')[0]['href']

'https://theconversation.com/curious-kids-why-is-our-dog-so-cute-138035'


soup.find_all('div', {'data-header-feature': '0'})[1].find_all('a')[0]['href']

'https://www.rover.com/blog/science-cute-dogs/'


soup.find_all('div', {'data-header-feature': '0'})[2].find_all('a')[0]['href']

'https://dogsnstuff.net/why-are-dogs-so-cute/'


for article in soup.find_all('div', {'data-header-feature': '0'}):
	articles.append(article.find_all('a')[0]['href'])


articles

['https://www.dabl.com/dabl-home/dabl-home/articles/scientific-reason-behind-why-we-find-dogs-so-cute#:~:text=In%20the%20past%2C%20research%20suggested,dogs%20are%20a%20perfect%20example.',
 'https://theconversation.com/curious-kids-why-is-our-dog-so-cute-138035',
 'https://www.rover.com/blog/science-cute-dogs/',
 'https://dogsnstuff.net/why-are-dogs-so-cute/',
 'https://www.owu.edu/news-media/from-our-perspective/why-is-my-dog-so-cute/',
 'https://www.wsj.com/story/national-dog-day-what-makes-your-pup-so-cute-its-all-in-the-eyebrows-feff3811',
 'https://www.sciencefocus.com/nature/why-do-we-find-puppies-so-cute/',
 'https://www.quora.com/What-makes-a-dog-cute-to-a-human-being',
 'https://www.psychologytoday.com/us/blog/thinking-about-kids/202003/why-are-puppies-so-cute',
 'https://vcahospitals.com/know-your-pet/what-your-dogs-cute-behaviors-really-mean']


articles_df = pd.concat([pd.Series(titles), pd.Series(articles)], axis = 1)
articles_df.columns = ['title', 'url']
articles_df


url = articles_df.iloc[0].url


driver = webdriver.Chrome()
driver.get(url)
html_data = driver.page_source
soup = BeautifulSoup(html_data, 'html.parser')


title = soup.find_all('h1')[0].text
title

'The Scientific Reason Behind Why We Find Dogs So Cute '


soup.find_all('meta')

[<meta content="#e70058" name="theme-color"/>,
 <meta content="width=device-width" name="viewport"/>,
 <meta charset="utf-8"/>,
 <meta content="max-image-preview:large" name="robots"/>,
 <meta content="summary_large_image" name="twitter:card"/>,
 <meta content="The Scientific Reason Behind Why We Find Dogs So Cute " name="description"/>,
 <meta content="https://www.dabl.com/dabl-home/dabl-home/articles/scientific-reason-behind-why-we-find-dogs-so-cute" property="og:url"/>,
 <meta content="The Scientific Reason Behind Why We Find Dogs So Cute " property="og:title"/>,
 <meta content="https://api.dabl.com/sites/default/files/styles/video_1280x720/public/images/2022-05/istock-1170679242.jpg?h=c549983f" property="og:image"/>,
 <meta content="The Scientific Reason Behind Why We Find Dogs So Cute " property="og:description"/>,
 <meta content="https://www.dabl.com/dabl-home/dabl-home/articles/scientific-reason-behind-why-we-find-dogs-so-cute" name="twitter:url"/>,
 <meta content="1280" name="twitter:image:width"/>,
 <meta content="720" name="twitter:image:height"/>,
 <meta content="https://api.dabl.com/sites/default/files/styles/video_1280x720/public/images/2022-05/istock-1170679242.jpg?h=c549983f" name="twitter:image"/>,
 <meta content="" name="keywords"/>,
 <meta content="The Scientific Reason Behind Why We Find Dogs So Cute " name="twitter:title"/>,
 <meta content="A recent study revealed the science behind why humans find dogs so adorable, and it’s at least partially our own fault! " name="twitter:description"/>,
 <meta content="The Scientific Reason Behind Why We Find Dogs So Cute " property="og:title"/>,
 <meta content="A recent study revealed the science behind why humans find dogs so adorable, and it’s at least partially our own fault! " property="og:description"/>,
 <meta content="21" name="next-head-count"/>,
 <meta content="A751Xsk4ZW3DVQ8WZng2Dk5s3YzAyqncTzgv+VaE6wavgTY0QHkDvUTET1o7HanhuJO8lgv1Vvc88Ij78W1FIAAAAAB7eyJvcmlnaW4iOiJodHRwczovL3d3dy5nb29nbGV0YWdtYW5hZ2VyLmNvbTo0NDMiLCJmZWF0dXJlIjoiUHJpdmFjeVNhbmRib3hBZHNBUElzIiwiZXhwaXJ5IjoxNjgwNjUyNzk5LCJpc1RoaXJkUGFydHkiOnRydWV9" http-equiv="origin-trial"/>,
 <meta content="A751Xsk4ZW3DVQ8WZng2Dk5s3YzAyqncTzgv+VaE6wavgTY0QHkDvUTET1o7HanhuJO8lgv1Vvc88Ij78W1FIAAAAAB7eyJvcmlnaW4iOiJodHRwczovL3d3dy5nb29nbGV0YWdtYW5hZ2VyLmNvbTo0NDMiLCJmZWF0dXJlIjoiUHJpdmFjeVNhbmRib3hBZHNBUElzIiwiZXhwaXJ5IjoxNjgwNjUyNzk5LCJpc1RoaXJkUGFydHkiOnRydWV9" http-equiv="origin-trial"/>]


title_metadata = [x.attrs for x in soup.find_all('meta') if 'og:title' in list(x.attrs.values())]
title = title_metadata[0]['content']
title

'The Scientific Reason Behind Why We Find Dogs So Cute '


url_metadata = [x.attrs for x in soup.find_all('meta') if 'og:url' in list(x.attrs.values())]
url = url_metadata[0]['content']
url

'https://www.dabl.com/dabl-home/dabl-home/articles/scientific-reason-behind-why-we-find-dogs-so-cute'


display.IFrame(url, '100%', 400)


image_metadata = [x.attrs for x in soup.find_all('meta') if 'og:image' in list(x.attrs.values())]
image = image_metadata[0]['content']
image

'https://api.dabl.com/sites/default/files/styles/video_1280x720/public/images/2022-05/istock-1170679242.jpg?h=c549983f'


display.Image(data=image.replace('?h=c549983f', ''), 
			  width = 600)


soup.find_all('div', {'class': 'content__presented-by'})[0]

<div class="content__presented-by"><div class="content__show-badge"><img alt="Dabl At Home Dec 2020" height="144" src="https://api.dabl.com/sites/default/files/styles/144x144/public/images/2020-12/DablAtHome_0.jpg?h=df1a205b" width="144"/></div><div>Presented By<br/><span>Dabl at Home</span></div></div>


author = soup.find_all('div', {'class': 'content__presented-by'})[0]\
						       .text.replace('Presented By', '')
author

'Dabl at Home'


article_text = soup.find_all('div', 
							 {'class': 'content__body-main'})[0]\
							  .get_text(strip=True).replace('\u200b', '')

article_text

'A recent study revealed the science behind why humans find dogs so adorable, and it’s at least partially our own fault!It’s no secret that dogs are adorable, and most pet owners would argue that their furry friend is the cutest in existence! But have you ever wondered what it is about your dog’s appearance that makes him so irresistible? Scientists have recently uncovered a genetic reason behind your fur baby’s endearing puppy dog eyes, but we’re going to have to go back in time many years to fully explain it.It’s thought that our bond with wolves — the ancestors of domesticated dogs — began approximately 40,000 years when most humans lived as hunter-gatherers. Dogs weren’t born until humans startedselectively breeding wolves with the most desirable traits an estimated 33,000 years agoto act as their companions. Thus, the dogs we know and love today were born and man gained a new best friend, forming a bond between our species that has withstood thousands of years.But anew studyhas found that humanity has imparted more on dogs than we initially realized, and it’s a key factor in why we find them so cute. In an effort to communicate with their human counterparts — and as a result of selectively breeding dogs whose facial expressions looked similar to our own — dogs have learned to mirror the expressions of their owners. Consequently, the looks our dogs mimic are what makes us fall in love with their appearances.Specifically, dogsdeveloped different facial musclesfrom wolves that allow them to replicate human expressions through eyebrow movement, with the evolutionary purpose of making themselves more endearing and communicating with their owners. While wolves primarily utilize “slow-twitch” muscle fibers that are useful for long, extended movements like howling, dogs predominantly use “fast-twitch” muscle fibers. These muscles allow dogs to mimic the facial expressions of humans when they’re sad, and to make their eyes appear larger, giving them an almost childlike appearance that really makes those puppy dog eyes sweep us off our feet. The study also found that dogs are unique from other mammals in that they are the only species that can demonstrate the bond they share with their pet parent through a mutual gaze.In the past, research suggested we found dogs to be cute because they subconsciously reminded us of babies and we are instinctually hardwired to want to care for and protect our young. It’s aphenomenon called kinderschema, of which dogs are a perfect example. Kinderschema describes mammals with large rounded eyes, small noses, and disproportionate head sizes with little mouths and chins that we find adorable because it reminds us of our own babies. This new study is one of the first times science has really investigated the role our dog’s facial anatomy plays in dog-human interactions and in developing affection toward our pets, and we areherefor it!Get more great inspiration, ideas, and pet parent tips on Dabl! Check ourDabl TV scheduleand find outwhere to watch Dabl TV.Related:How Does The Weather Impact Your Pet’s Mental Health?Related:New Puppy Checklist: What To Buy Before Bringing Your New Pet HomeRelated:6 Reasons To Become A Foster Home For Pets In Need!Dogs'


article_text = ' '.join([p.get_text(strip=True) for p in soup.find_all('p')])
article_text

'It’s no secret that dogs are adorable, and most pet owners would argue that their furry friend is the cutest in existence! But have you ever wondered what it is about your dog’s appearance that makes him so irresistible? Scientists have recently uncovered a genetic reason behind your fur baby’s endearing puppy dog eyes, but we’re going to have to go back in time many years to fully explain it. It’s thought that our bond with wolves — the ancestors of domesticated dogs — began approximately 40,000 years when most humans lived as hunter-gatherers. Dogs weren’t born until humans startedselectively breeding wolves with the most desirable traits an estimated 33,000 years agoto act as their companions. Thus, the dogs we know and love today were born and man gained a new best friend, forming a bond between our species that has withstood thousands of years. But anew studyhas found that humanity has imparted more on dogs than we initially realized, and it’s a key factor in why we find them so cute. In an effort to communicate with their human counterparts — and as a result of selectively breeding dogs whose facial expressions looked similar to our own — dogs have learned to mirror the expressions of their owners. Consequently, the looks our dogs mimic are what makes us fall in love with their appearances. Specifically, dogsdeveloped different facial musclesfrom wolves that allow them to replicate human expressions through eyebrow movement, with the evolutionary purpose of making themselves more endearing and communicating with their owners. While wolves primarily utilize “slow-twitch” muscle fibers that are useful for long, extended movements like howling, dogs predominantly use “fast-twitch” muscle fibers. These muscles allow dogs to mimic the facial expressions of humans when they’re sad, and to make their eyes appear larger, giving them an almost childlike appearance that really makes those puppy dog eyes sweep us off our feet. The study also found that dogs are unique from other mammals in that they are the only species that can demonstrate the bond they share with their pet parent through a mutual gaze. In the past, research suggested we found dogs to be cute because they subconsciously reminded us of babies and we are instinctually hardwired to want to care for and protect our young. It’s aphenomenon called \u200b\u200bkinderschema, of which dogs are a perfect example. Kinderschema describes mammals with large rounded eyes, small noses, and disproportionate head sizes with little mouths and chins that we find adorable because it reminds us of our own babies. This new study is one of the first times science has really investigated the role our dog’s facial anatomy plays in dog-human interactions and in developing affection toward our pets, and we areherefor it! Get more great inspiration, ideas, and pet parent tips on Dabl! Check ourDabl TV scheduleand find outwhere to watch Dabl TV.  '


article = pd.DataFrame([title, author, article_text]).T
article.columns = ['title', 'author', 'article_text']
article

	title	url
0	The Scientific Reason Behind Why We Find Dogs ...	https://www.dabl.com/dabl-home/dabl-home/artic...
1	Curious Kids: Why is our dog so cute? - The Co...	https://theconversation.com/curious-kids-why-i...
2	The Science of Cute: How Cuteness Makes Us Lov...	https://www.rover.com/blog/science-cute-dogs/
3	Why Are Dogs So Cute? - Dogs 'N Stuff	https://dogsnstuff.net/why-are-dogs-so-cute/
4	Why is my dog so cute? \| Ohio Wesleyan University	https://www.owu.edu/news-media/from-our-perspe...
5	Science Explains What Makes Your Dog So Cute -...	https://www.wsj.com/story/national-dog-day-wha...
6	Why do we find puppies so cute? - BBC Science ...	https://www.sciencefocus.com/nature/why-do-we-...
7	What makes a dog cute to a human being? - Quora	https://www.quora.com/What-makes-a-dog-cute-to...
8	Why Are Puppies So Cute? \| Psychology Today	https://www.psychologytoday.com/us/blog/thinki...
9	What Your Dog's Cute Behaviors Really Mean	https://vcahospitals.com/know-your-pet/what-yo...

🕸️ Web Scraping with Selenium and Beautiful Soup¶

Driver instantiation and retrieving HTML data¶

Making soup¶

Getting the title¶

Getting the author¶

Getting the article text¶

Google Search¶

Accessing the search bar¶

Inputting the search query and hitting "enter"¶

Collecting the HTML data for the results of the search¶

Making soup¶

Getting All Article Titles¶

Drop the two non-titles from the end of the list¶

Remove the 4 extra titles between header article and other articles¶

Getting head article¶

Checking the number of articles in the heading¶

Getting the rest of the articles¶

Combine article titles and urls as a dataframe¶

Scraping the first article in the dataframe¶

The title¶

What can be found in the meta data?¶

Title via meta data¶

URL via meta data¶

Image via meta data¶

Getting author data¶

First method of getting article text¶

Second method of getting article text¶

Compiling the data as a dataframe¶