from bs4 import BeautifulSoup as soup
from urllib.request import urlopen as uReq
import urllib.request
import re

lists = open("AmiAmi_mini.csv").read().split("\n")[:-1]

filename = "Ami_full.csv"
f = open(filename, "w", encoding="utf-8")
headers = "fig_name_eng, fig_name_jap, Scale, Size, Type, Materials, Sell_price, Price, Sale, Status, Code, date, brand, line_prod, series, char_name, sculpt\n"
f.write(headers)
f.close

for url in lists:

    uClient = uReq(url)
    page_html = uClient.read()
    uClient.close()
    page_soup = soup(page_html, "html.parser")

    page_soup = soup(page_html, "html.parser")
    page_soup.findAll("div", style="width:600px;")

    f = open(filename, "a", encoding="utf-8")

    info4 = page_soup.findAll("h2", {"class": "heading_10"})
    container4 = info4[0]

    for container4 in info4:
        jap = container4.find('span', class_='').text
        if jap:
            Jap = jap.strip()
        else:
            Jap = "NA"
        r = str(container4)
        container5 = soup(r[:r.rindex('<br/>')], 'html.parser')
        if container5:
            eng = container5.find('h2').text.strip()
        else:
            eng = "NA"

        f.write(eng.replace(",", "|") + "," + jap.replace(",", "|") + ",")

    info1 = page_soup.findAll("p", {"class": "box_01"})
    container1 = info1[0]

    for container1 in info1:
        Scale1 = container1.findAll(text=re.compile('Scale'))
        if Scale1:
            Scale = Scale1[0].strip(' \t\n\r')
        else:
            Scale = "NA"

        Size1 = container1.findAll(text=re.compile('Size'))
        if Size1:
            Size = Size1[0].strip(' \t\n\r')
        else:
            Size = "NA"

        Type1 = info1[0].next_element.strip()
        if Type1:
            Type = Type1
        else:
            Type = "NA"

        Material1 = container1.findAll(text=re.compile('Material'))
        if Material1:
            Material = Material1[0].strip(' \t\n\r')
        else:
            Material = "NA"

        print("Scale: "+Scale)
        print("Size: "+Size)
        print("Type: "+Type)
        print("Materials: "+Material)
        f.write(Scale.replace(",", ".") + "," + Size.replace(",", ".") + "," + Type + "," + Material + ",")

    info2 = page_soup.findAll("ul")
    container2 = info2[4]

    for container2 in info2:
        Sell_price1 = container2.findAll("li", {"class": "selling_price"})
        if Sell_price1:
            Sell_price = Sell_price1[0].text.strip()
        else:
            Sell_price = "NA"

        Price1 = container2.findAll("li", {"class": "price"})
        if Price1:
            Price = Price1[0].findAll(text=re.compile('JPY'))[0].strip()
        else:
            Price = "NA"

        sale1 = container2.li("span", {"class": "off_price"})
        if sale1:
            Sale = sale1[0].text
        else:
            Sale = "NA"

        Status = container2.findAll("li", {"class": "selling_price"})
        if Status:
            Status = Status[1].text.strip()
        else:
            Status = "NA"

        print("Sell price: "+Sell_price)
        print("Price: "+ Price)
        print("Sale: "+Sale)
        print("Status: "+Status)
        f.write(Sell_price.replace(",", ".") + "," + Price.replace(",", ".") + "," + Sale.replace(",", ".") + "," + Status.replace(",", ".") + ",")

    info3 = page_soup.findAll("dl", {"class": "spec_data"})
    container3 = info3[0]

    for container3 in info3:
        code = container3.find('dt', text='JAN Code')
        if code:
            Code = code.find_next_sibling("dd").text.strip()
        else:
            Code = "NA"

        Release_Date = container3.find('dt', text='Release Date')
        if Release_Date:
            date = Release_Date.find_next_sibling("dd").text.strip()
        else:
            date = "NA"

        Brand = container3.find('dt', text='Brand')
        if Brand:
            brand = Brand.find_next_sibling("dd").text.strip()
        else:
            brand = "NA"

        Line = container3.find('dt', text='Product Line')
        if Line:
            line_prod = Line.find_next_sibling("dd").text.strip()
        else:
            line_prod = "NA"

        Series = container3.find('dt', text='Series Title')
        if Series:
            series = Series.find_next_sibling("dd").text.strip()
        else:
            series = "NA"

        Name = container3.find('dt', text='Character Name')
        if Name:
            char_name = Name.find_next_sibling("dd").text.strip()
        else:
            char_name = "NA"

        Sculptor = container3.find('dt', text='Sculptor')
        if Sculptor:
            sculpt = Sculptor.find_next_sibling("dd").text.strip()
        else:
            sculpt = "NA"

        f.write(Code.replace(",", ".") + "," + date + "," + brand.replace(",", ".") + "," + line_prod.replace(",", ".") + "," + series.replace(",", ".") + "," + char_name.replace(",", ".") + "," + sculpt + "\n")


    f.close