import time
import sys
import re
import requests
from tqdm import trange
from multiprocessing import Queue, Manager, Process


re_wp = re.compile(r'\/wp-(admin|content|includes)\/', flags=re.IGNORECASE|re.UNICODE)


def fetch(url):
    try:
        headers = {'host': url}
        response = requests.get('http://' + url, timeout=(3.05, 10.0), headers=headers)
        result = 1 if re_wp.search(response.text) else 0

    except Exception as e:
        result = 'e'

    return '{}|{}\n'.format(url, result)


def worker(urls, results):
    while True:
        try:
            url = urls.get(timeout=15)
        except Exception:
            break

        results.put(fetch(url))


def data_loader(urls, queue):
    for url in urls:
        if queue.qsize() > 5000:
            time.sleep(10)

        queue.put(url)


def main(in_file, out_file, workers=200):
    with open(in_file, 'r') as f:
        urls = [url for url in map(str.strip, f) if url]

    with Manager() as man:
        queue = man.Queue()
        results = man.Queue()

        Process(target=data_loader, args=(urls, queue)).start()

        for _ in range(workers):
            Process(target=worker, args=(queue, results), daemon=True).start()

        with open(out_file, 'w') as out:
            for i in trange(len(urls), smoothing=0.01, file=sys.stdout):
                try:
                    result = results.get(timeout=5 * 60)
                except:
                    break

                out.write(result)


if __name__ == '__main__':
    main('xal.txt', 'wp_xal.txt')