#!
/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 28 11:35:04 2020
@author: chrislovejoy
"""
import urllib
import requests
from bs4 import BeautifulSoup
import selenium
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
import pandas as pd
import os
def find_jobs_from(website, job_title, location, desired_characs,
filename="results.xls"):
if website == 'Indeed':
job_soup = load_indeed_jobs_div(job_title, location)
jobs_list, num_listings = extract_job_information_indeed(job_soup,
desired_characs)
if website == 'CWjobs':
location_of_driver = os.getcwd()
driver = initiate_driver(location_of_driver, browser='chrome')
job_soup = make_job_search(job_title, location, driver)
jobs_list, num_listings = extract_job_information_cwjobs(job_soup,
desired_characs)
save_jobs_to_excel(jobs_list, filename)
print('{} new job postings retrieved from {}. Stored in
{}.'.format(num_listings,
website,
filename))
## ======================= GENERIC FUNCTIONS ======================= ##
def save_jobs_to_excel(jobs_list, filename):
jobs = pd.DataFrame(jobs_list)
jobs.to_excel(filename)
## ================== FUNCTIONS FOR INDEED.CO.UK =================== ##
def load_indeed_jobs_div(job_title, location):
getVars = {'q' : job_title, 'l' : location, 'fromage' : 'last', 'sort' :
'date'}
url = ('https://hk.indeed.com/' + urllib.parse.urlencode(getVars))
page = requests.get(url)
soup = BeautifulSoup(page.content, "html.parser")
job_soup = soup.find(id="resultsCol")
return job_soup
def extract_job_information_indeed(job_soup, desired_characs):
job_elems = job_soup.find_all('div', class_='jobsearch-SerpJobCard')
cols = []
extracted_info = []
if 'titles' in desired_characs:
titles = []
cols.append('titles')
for job_elem in job_elems:
titles.append(extract_job_title_indeed(job_elem))
extracted_info.append(titles)
if 'companies' in desired_characs:
companies = []
cols.append('companies')
for job_elem in job_elems:
companies.append(extract_company_indeed(job_elem))
extracted_info.append(companies)
if 'links' in desired_characs:
links = []
cols.append('links')
for job_elem in job_elems:
links.append(extract_link_indeed(job_elem))
extracted_info.append(links)
if 'date_listed' in desired_characs:
dates = []
cols.append('date_listed')
for job_elem in job_elems:
dates.append(extract_date_indeed(job_elem))
extracted_info.append(dates)
jobs_list = {}
for j in range(len(cols)):
jobs_list[cols[j]] = extracted_info[j]
num_listings = len(extracted_info[0])
return jobs_list, num_listings
def extract_job_title_indeed(job_elem):
title_elem = job_elem.find('h2', class_='title')
title = title_elem.text.strip()
return title
def extract_company_indeed(job_elem):
company_elem = job_elem.find('span', class_='company')
company = company_elem.text.strip()
return company
def extract_link_indeed(job_elem):
link = job_elem.find('a')['href']
link = 'www.Indeed.co.uk/' + link
return link
def extract_date_indeed(job_elem):
date_elem = job_elem.find('span', class_='date')
date = date_elem.text.strip()
return date
## ================== FUNCTIONS FOR CWJOBS.CO.UK =================== ##
def initiate_driver(location_of_driver, browser):
if browser == 'chrome':
driver = webdriver.Chrome(executable_path=(location_of_driver +
"/chromedriver"))
elif browser == 'firefox':
driver = webdriver.Firefox(executable_path=(location_of_driver +
"/firefoxdriver"))
elif browser == 'safari':
driver = webdriver.Safari(executable_path=(location_of_driver +
"/safaridriver"))
elif browser == 'edge':
driver = webdriver.Edge(executable_path=(location_of_driver +
"/edgedriver"))
return driver
def make_job_search(job_title, location, driver):
driver.get('https://www.cwjobs.co.uk/')
# Select the job box
job_title_box = driver.find_element_by_name('Keywords')
# Send job information
job_title_box.send_keys(job_title)
# Selection location box
location_box = driver.find_element_by_id('location')
# Send location information
location_box.send_keys(location)
# Find Search button
search_button = driver.find_element_by_id('search-button')
search_button.click()
driver.implicitly_wait(5)
page_source = driver.page_source
job_soup = BeautifulSoup(page_source, "html.parser")
return job_soup
def extract_job_information_cwjobs(job_soup, desired_characs):
job_elems = job_soup.find_all('div', class_="job")
cols = []
extracted_info = []
if 'titles' in desired_characs:
titles = []
cols.append('titles')
for job_elem in job_elems:
titles.append(extract_job_title_cwjobs(job_elem))
extracted_info.append(titles)
if 'companies' in desired_characs:
companies = []
cols.append('companies')
for job_elem in job_elems:
companies.append(extract_company_cwjobs(job_elem))
extracted_info.append(companies)
if 'links' in desired_characs:
links = []
cols.append('links')
for job_elem in job_elems:
links.append(extract_link_cwjobs(job_elem))
extracted_info.append(links)
if 'date_listed' in desired_characs:
dates = []
cols.append('date_listed')
for job_elem in job_elems:
dates.append(extract_date_cwjobs(job_elem))
extracted_info.append(dates)
jobs_list = {}
for j in range(len(cols)):
jobs_list[cols[j]] = extracted_info[j]
num_listings = len(extracted_info[0])
return jobs_list, num_listings
def extract_job_title_cwjobs(job_elem):
title_elem = job_elem.find('h2')
title = title_elem.text.strip()
return title
def extract_company_cwjobs(job_elem):
company_elem = job_elem.find('h3')
company = company_elem.text.strip()
return company
def extract_link_cwjobs(job_elem):
link = job_elem.find('a')['href']
return link
def extract_date_cwjobs(job_elem):
link_elem = job_elem.find('li', class_='date-posted')
link = link_elem.text.strip()
return link
