fork download
copy 
  1. from selenium import webdriver
  2. from selenium.webdriver.common.by import By
  3. from selenium.webdriver.common.keys import Keys
  4. from selenium.webdriver.support.ui import Select
  5. from selenium.webdriver.support.ui import WebDriverWait
  6. from selenium.common.exceptions import TimeoutException
  7. from selenium.webdriver.support import expected_conditions as EC
  8. from selenium.common.exceptions import NoSuchElementException
  9. from selenium.common.exceptions import NoAlertPresentException
  10. import sys
  11. import csv
  12. import requests
  13. import time, re
  14.  
  15. from bs4 import BeautifulSoup
  16.  
  17. browser = webdriver.Firefox()
  18. BASE_URL = 'http://a...content-available-to-author-only...s.com/'
  19. accept_next_alert = True
  20.  
  21.  
  22. def getting_titles(url):
  23.     source_code = requests.get(url)
  24.     plain_text = source_code.text
  25.     soup = BeautifulSoup(plain_text, 'html.parser')
  26.     for link in soup.findAll('h3', {'class': 'grid-col__h3 grid-col__h3--recipe-grid'}):
  27.         title = link.string
  28.  
  29.         return title
  30.  
  31.  
  32. def save(recipes, path, fieldname):
  33.     with open(path, 'w') as csvfile:
  34.         fieldnames = ['Titles']
  35.         writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
  36.         writer.writeheader()
  37.         writer.writerow({'Titles': '#'})
  38.  
  39.  
  40. def main(url=BASE_URL):
  41.     i = 0
  42.     for i in range(1, 3):
  43.         i += 1
  44.         driver = browser
  45.         delay = 5
  46.         paggination = '?page=' + str(i)
  47.         driver.get(BASE_URL + paggination)
  48.         browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
  49.         time.sleep(10)
  50.         html_source = BASE_URL
  51.         data = html_source.encode('utf-8')
  52.         getting_titles(url + paggination)
  53.  
  54.  
  55. if __name__ == "__main__":
  56.     main()
Runtime error #stdin #stdout #stderr 0.02s 9936KB
comments (?)
stdin
copy 
Standard input is empty
stdout
copy 
Standard output is empty
stderr
copy 
Traceback (most recent call last):
  File "./prog.py", line 1, in <module>
ImportError: No module named 'selenium'
https://ideone.com/YcRQus
language:
Python 3 (python  3.9.5)
created:
8 years ago
visibility:
public

Share or Embed source code

Discover > Sphere Engine API

The brand new service which powers Ideone!

Discover > IDE Widget

Widget for compiling and running the source code in a web browser!