import scrapy
import json

class RTSpider(scrapy.Spider):
	name = "RTSpider"
	start_urls = ['https://r...content-available-to-author-only...r.cr/forum/index.php?map=1']

	def parse(self, response):

		def getTree(node):
			nonlocal raw_map
			node_parent = node
			tree = {}
			ul_amt = range(len(raw_map.css(node+">*")))
			node += ">ul:nth-child({})>li:nth-child({})"
			for ul_position in ul_amt:
				li_amt = range(len(raw_map.css(
					node_parent+">ul:nth-child({})>*".format(ul_position+1)))
				)
				for li_position in li_amt:
					nodeaddr = node.format(ul_position+1, li_position+1)
					if raw_map.css(nodeaddr): 
						cat_node_name = raw_map.css(nodeaddr + ">span>span::attr(title)").extract_first()
						forum_node_name = raw_map.css(nodeaddr + ">span a::text").extract_first()
						node_name = forum_node_name if forum_node_name else cat_node_name
						link = raw_map.css(nodeaddr + ">span a::attr(href)").extract_first()
						tree[node_name] = {"link": link, "subtree": getTree(nodeaddr)}
			return tree

		raw_map = response.css("#f-map")
		map = getTree("#f-map")
		with open("crack.json", 'w', encoding='utf-8') as f:
			f.write(json.dumps(map, indent=4, ensure_ascii=False))