fork download
copy 
  1. # -*- coding: utf-8 -*-
  2.  
  3. '''Async checker who take url's from input file, where one url in string,
  4. create async queue where one url relates to one proxies - one item in queue,
  5. get responses from server through http proxy and sort in output files.
  6. '''
  7. import logging
  8. from eventlet import GreenPool
  9. from erequests import AsyncRequestFactory
  10.  
  11. urls_and_proxies = []
  12. timeout_for_req = 10.001        # exception if no bytes have been received
  13.                                 # on the underlying socket for timeout seconds
  14. timeout_for_proxy_test = 15.0   # 15 sec
  15. url_for_proxy_tests = 'http://content-available-to-author-only.ru'
  16. proxies_file_name = 'proxy.txt' # one proxy in string http://109.202.19.26:8080
  17. bad_proxies_file_name = 'bad_proxy.txt'
  18. input_file_name = 'input.txt'
  19. output_file_names = ['1xx.txt', '2xx.txt', '3xx.txt',
  20.                     '4xx.txt', '5xx.txt', 'error.txt']
  21. logging.basicConfig(
  22.     level=logging.WARNING, format='%(asctime)s - %(levelname)s - %(message)s')
  23.  
  24. class ProxyError(Exception):
  25.     pass
  26.  
  27.  
  28. def test_proxies_from_file(proxies_file_name):
  29.     '''The function create and return list proxy form file,
  30. check and if they do not work - excludes from the list and from file
  31. '''
  32.     # Get http proxies from file
  33.     logging.warning('test proxies')
  34.     proxy_from_response = []
  35.     proxy_checked = []
  36.     proxy_from_file = [proxy.strip() for proxy in open(proxies_file_name)]
  37.     requests = async_reqest(
  38.         [(url_for_proxy_tests, { 'http': proxy }) for proxy in proxy_from_file],
  39.             number_of_reqests=len(proxy_from_file),
  40.             timeout_for_req=timeout_for_proxy_test)
  41.     # Get only good proxy
  42.     for response in requests:
  43.         try:
  44.             response.status_code
  45.         except AttributeError:
  46.             pass
  47.         else:
  48.             proxy_from_response.append(
  49.                 response.connection.proxy_manager.keys()[0])
  50.     for from_response in proxy_from_response:
  51.         for from_file in proxy_from_file:
  52.             if from_response in from_file:
  53.                 proxy_checked.append(from_file)
  54.                 break
  55.     # Write bad proxy in file or rise exception no live proxies in file
  56.     if not proxy_checked:
  57.         raise ProxyError, 'Run out proxy'
  58.     elif len(proxy_checked) != len(proxy_from_file):
  59.         with open(bad_proxies_file_name, 'w') as proxies_file:
  60.             for proxy in [bad_proxy for bad_proxy in proxy_from_file 
  61.                             if bad_proxy not in proxy_checked]:
  62.                 proxies_file.write(proxy+'\n')
  63.     logging.warning('end test proxies')
  64.     return proxy_checked
  65.  
  66. def imap(requests, size=10):
  67.     '''Concurrently converts a sequence of AsyncRequest objects to Responses.
  68. This function from erequests where I added url in exception object.
  69. '''
  70.  
  71.     pool = GreenPool(size)
  72.  
  73.     def send(r):
  74.         try:
  75.             return r.send()
  76.         except Exception as e:
  77.             # Add url in exception object because in message I only see proxy
  78.             e.url = r.url
  79.             return e
  80.  
  81.     for r in pool.imap(send, requests):
  82.         yield r
  83.  
  84. def sort_responses_to_files(requests):
  85.     '''Get first number from HTTP status code and write response
  86. in file where first letter in name is this number, 
  87. or if response is exception object - write in error.txt
  88. '''
  89.     logging.warning('get and sort responses')
  90.     for response in requests:
  91.         try:
  92.             response.status_code
  93.         except AttributeError:
  94.             # Write in error.txt url and exception
  95.             logging.warning('%s | %s' % (response.url, response))
  96.             output_files['error.txt'
  97.                 ].write('%s | %s\n' % (response.url, response))
  98.         else:
  99.             # Write status and url in output files)
  100.             logging.warning('%s | %s' % (response.status_code, response.url))
  101.             file_name = str(response.status_code)[0] + 'xx.txt'
  102.             if output_files.has_key(file_name):
  103.                 output_files[file_name
  104.                     ].write('%s | %s\n' % (response.status_code, response.url))
  105.             else:
  106.                 output_files['error.txt'
  107.                     ].write('%s | %s\n' % (response.url, response))
  108.  
  109. def async_reqest(urls_and_proxies,
  110.                 number_of_reqests,
  111.                 timeout_for_req=timeout_for_req):
  112.     '''Resive list where item is (url, proxy={ 'http': 'ip:port' })
  113. and returns acync queue of reqests.
  114. '''
  115.     logging.warning('create async requests')
  116.     requests = imap(
  117.                 (AsyncRequestFactory.head(
  118.                     url, timeout=timeout_for_req, proxies=proxy)
  119.                         for url, proxy in urls_and_proxies), number_of_reqests)
  120.     return requests
  121.  
  122. if __name__ == '__main__':
  123.     # Delete old if exist and create new output files
  124.     output_files = dict(
  125.         [(file_name, open(file_name, 'w')) for file_name in output_file_names])
  126.     # Get and test all proxies from file
  127.     proxies_list = test_proxies_from_file(proxies_file_name)
  128.     with open(input_file_name) as input_file:
  129.         logging.warning('create list urls for async reqest')
  130.         for url in input_file:
  131.             # Add urls in queue from file, while exist one proxy for each url
  132.             if proxies_list:
  133.                 proxy = { 'http': proxies_list[0]}
  134.                 url = url.strip()
  135.                 urls_and_proxies.append((url, proxy))
  136.                 del proxies_list[0]
  137.             else:
  138.                 # If run out proxy - request and write result
  139.                 logging.warning('run queue')
  140.                 requests = async_reqest(urls_and_proxies,
  141.                                         number_of_reqests=len(urls_and_proxies))
  142.                 sort_responses_to_files(requests)
  143.                 urls_and_proxies = []
  144.                 # Get again all proxies from file and test
  145.                 proxies_list = test_proxies_from_file(proxies_file_name)
  146.                 logging.warning('create list urls for async reqest')
  147.         else:
  148.             # For remainder urls
  149.             logging.warning('run remainder queue')
  150.             requests = async_reqest(urls_and_proxies,
  151.                                     number_of_reqests=len(urls_and_proxies))
  152.             sort_responses_to_files(requests)
Runtime error #stdin #stdout #stderr 0.03s 8512KB
comments (?)
stdin
copy 
Standard input is empty
stdout
copy 
Standard output is empty
stderr
copy 
Traceback (most recent call last):
  File "prog.py", line 8, in <module>
ImportError: No module named eventlet
https://ideone.com/uxW98R
language:
Python (cpython 2.7.16)
created:
11 years ago
visibility:
public

Share or Embed source code

Discover > Sphere Engine API

The brand new service which powers Ideone!

Discover > IDE Widget

Widget for compiling and running the source code in a web browser!