from bs4 import BeautifulSoup as soup
from urllib.request import urlopen as uReq
import urllib.request
import re

lists = open("AmiAmi_mini.csv").read().split("\n")[:-1]

filename = "Ami_full.csv"
f = open(filename, "w", encoding="utf-8")
headers = "fig_name_eng, fig_name_jap, Scale, Size, Type, Materials, Sell_price, Price, Sale, Status, Code, date, brand, line_prod, series, char_name, sculpt\n"
f.write(headers)
f.close

for url in lists:

    uClient = uReq(url)
    page_html = uClient.read()
    uClient.close()
    page_soup = soup(page_html, "html.parser")

    page_soup = soup(page_html, "html.parser")
    page_soup.findAll("div", style="width:600px;")

    f = open(filename, "a", encoding="utf-8")

    info4 = page_soup.findAll("h2", {"class": "heading_10"})
    container4 = info4[0]

    for container4 in info4:
        jap = container4.find('span', class_='').text
        if jap:
            Jap = jap.strip()
        else:
            Jap = "NA"
        r = str(container4)
        container5 = soup(r[:r.rindex('<br/>')], 'html.parser')
        if container5:
            eng = container5.find('h2').text.strip()
        else:
            eng = "NA"

        f.write(eng.replace(",", "|") + "," + jap.replace(",", "|") + ",")

    info1 = page_soup.findAll("p", {"class": "box_01"})
    container1 = info1[0]

    for container1 in info1:
        Scale1 = container1.findAll(text=re.compile('Scale'))
        if Scale1:
            Scale = Scale1[0].strip(' \t\n\r')
        else:
            Scale = "NA"

        Size1 = container1.findAll(text=re.compile('Size'))
        if Size1:
            Size = Size1[0].strip(' \t\n\r')
        else:
            Size = "NA"

        Type1 = info1[0].next_element.strip()
        if Type1:
            Type = Type1
        else:
            Type = "NA"

        Material1 = container1.findAll(text=re.compile('Material'))
        if Material1:
            Material = Material1[0].strip(' \t\n\r')
        else:
            Material = "NA"

        print("Scale: "+Scale)
        print("Size: "+Size)
        print("Type: "+Type)
        print("Materials: "+Material)
        f.write(Scale.replace(",", ".") + "," + Size.replace(",", ".") + "," + Type + "," + Material + ",")

    info2 = page_soup.findAll("ul")
    container2 = info2[4]

    for container2 in info2:
        Sell_price1 = container2.findAll("li", {"class": "selling_price"})
        if Sell_price1:
            Sell_price = Sell_price1[0].text.strip()
        else:
            Sell_price = "NA"

        Price1 = container2.findAll("li", {"class": "price"})
        if Price1:
            Price = Price1[0].findAll(text=re.compile('JPY'))[0].strip()
        else:
            Price = "NA"

        sale1 = container2.li("span", {"class": "off_price"})
        if sale1:
            Sale = sale1[0].text
        else:
            Sale = "NA"

        Status = container2.findAll("li", {"class": "selling_price"})
        if Status:
            Status = Status[1].text.strip()
        else:
            Status = "NA"

        print("Sell price: "+Sell_price)
        print("Price: "+ Price)
        print("Sale: "+Sale)
        print("Status: "+Status)
        f.write(Sell_price.replace(",", ".") + "," + Price.replace(",", ".") + "," + Sale.replace(",", ".") + "," + Status.replace(",", ".") + ",")

    info3 = page_soup.findAll("dl", {"class": "spec_data"})
    container3 = info3[0]

	fields = ["JAN Code", "Release Date", "Brand", "Product Line",
				"Series Title", "Character Name", "Sculptor"]

    for container3 in info3:
    	data = []
    	for field in fileds:
    		tag = container3.find('dt', text=field)
    		if tag:
    			field_text = tag.find_next_sibling("dd").text.strip().replace(",",".")
    		else:
    			field_text = "NA"
    		data.append(field_text)

    	f.write(",".join(data))

    f.close()