#!/usr/bin/env python

import scraperwiki
import requests
from bs4 import BeautifulSoup
import string

allitems = []

uri = "http://l...content-available-to-author-only...g.com/r/discussion/new/"
html = requests.get(uri)
soup = BeautifulSoup(html.content)
items = soup.find_all(class_="post list")

while len(items) > 0:
    for item in items:
        
        if (item is not None) and (item.div is not None) and (item.div.span is not None):
            points = item.div.span.span
            itemdate = item.find(class_="date")
            id = string.split(item.h2.a['href'], '/')[4]
            title = item.h2.a.string
            if (points is not None) and (itemdate is not None):
                allitems.append([points.string, title, itemdate.string, id])
                unique_keys = [ 'id' ]
                data = { 'id':unicode(id), 'title':unicode(title), 'points':int(points.string), 'posted_on':unicode(itemdate.string)}
                scraperwiki.sql.save(unique_keys, data)
    
    uri = "http://l...content-available-to-author-only...g.com/r/discussion/new/?after=t3_" + allitems[-1][3]
    html = requests.get(uri)
    soup = BeautifulSoup(html.content)
    items = soup.find_all(class_="post list")
    
    print uri, len(allitems)