import os
import sys
import django
from django.core.files.storage import default_storage
from django.core.files.base import ContentFile
from django.db.utils import IntegrityError
os.environ["DJANGO_SETTINGS_MODULE"] = "WillBeams.settings"
sys.path.append("WillBeams/")
# os.environ["DJANGO_SETTINGS_MODULE"] = "valiz.settings"
# sys.path.append("valiz/")
django.setup()
from will_beams import models
import scraper
class DownloadToModel(scraper.Downloader):
def __init__(self, *args, **kwargs):
self._webm_obj = None
super().__init__(*args, **kwargs)
def _download(self, data):
md5 = data[-1]
try:
models.Webm.increase_rating(md5)
result = None, None, None
except models.Webm.DoesNotExist:
self._webm_obj = models.Webm(md5=md5)
result = super()._download(data)
return result
def save(self, filename, webm, thumb):
if not filename:
return
if webm:
filename += '.webm'
path = os.path.join(
models.get_media_folder(self._webm_obj, filename), filename)
webm_path = default_storage.save(path, ContentFile(webm))
self._webm_obj.video = webm_path
if thumb:
filename += '.jpg'
path = os.path.join(
models.get_media_folder(self._webm_obj, filename), filename)
thumb_path = default_storage.save(path, ContentFile(thumb))
self._webm_obj.thumbnail = thumb_path
try:
self._webm_obj.save()
except IntegrityError:
models.Webm.increase_rating(self._webm_obj.md5)
def work(self, *args, **kwargs):
return super().work(self.save)
if __name__ == '__main__':
sections = ['vg', 'b', 'a', 'mov']
worker = scraper.MainWorker(sections, downloader=DownloadToModel)
worker.work()
aW1wb3J0IG9zCmltcG9ydCBzeXMKCmltcG9ydCBkamFuZ28KZnJvbSBkamFuZ28uY29yZS5maWxlcy5zdG9yYWdlIGltcG9ydCBkZWZhdWx0X3N0b3JhZ2UKZnJvbSBkamFuZ28uY29yZS5maWxlcy5iYXNlIGltcG9ydCBDb250ZW50RmlsZQpmcm9tIGRqYW5nby5kYi51dGlscyBpbXBvcnQgSW50ZWdyaXR5RXJyb3IKCm9zLmVudmlyb25bIkRKQU5HT19TRVRUSU5HU19NT0RVTEUiXSA9ICJXaWxsQmVhbXMuc2V0dGluZ3MiCnN5cy5wYXRoLmFwcGVuZCgiV2lsbEJlYW1zLyIpCgojIG9zLmVudmlyb25bIkRKQU5HT19TRVRUSU5HU19NT0RVTEUiXSA9ICJ2YWxpei5zZXR0aW5ncyIKIyBzeXMucGF0aC5hcHBlbmQoInZhbGl6LyIpCgpkamFuZ28uc2V0dXAoKQpmcm9tIHdpbGxfYmVhbXMgaW1wb3J0IG1vZGVscwppbXBvcnQgc2NyYXBlcgoKCmNsYXNzIERvd25sb2FkVG9Nb2RlbChzY3JhcGVyLkRvd25sb2FkZXIpOgoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCAqYXJncywgKiprd2FyZ3MpOgogICAgICAgIHNlbGYuX3dlYm1fb2JqID0gTm9uZQogICAgICAgIHN1cGVyKCkuX19pbml0X18oKmFyZ3MsICoqa3dhcmdzKQoKICAgIGRlZiBfZG93bmxvYWQoc2VsZiwgZGF0YSk6CiAgICAgICAgbWQ1ID0gZGF0YVstMV0KICAgICAgICB0cnk6CiAgICAgICAgICAgIG1vZGVscy5XZWJtLmluY3JlYXNlX3JhdGluZyhtZDUpCiAgICAgICAgICAgIHJlc3VsdCA9IE5vbmUsIE5vbmUsIE5vbmUKICAgICAgICBleGNlcHQgbW9kZWxzLldlYm0uRG9lc05vdEV4aXN0OgogICAgICAgICAgICBzZWxmLl93ZWJtX29iaiA9IG1vZGVscy5XZWJtKG1kNT1tZDUpCiAgICAgICAgICAgIHJlc3VsdCA9IHN1cGVyKCkuX2Rvd25sb2FkKGRhdGEpCgogICAgICAgIHJldHVybiByZXN1bHQKCiAgICBkZWYgc2F2ZShzZWxmLCBmaWxlbmFtZSwgd2VibSwgdGh1bWIpOgogICAgICAgIGlmIG5vdCBmaWxlbmFtZToKICAgICAgICAgICAgcmV0dXJuCgogICAgICAgIGlmIHdlYm06CiAgICAgICAgICAgIGZpbGVuYW1lICs9ICcud2VibScKICAgICAgICAgICAgcGF0aCA9IG9zLnBhdGguam9pbigKICAgICAgICAgICAgICAgIG1vZGVscy5nZXRfbWVkaWFfZm9sZGVyKHNlbGYuX3dlYm1fb2JqLCBmaWxlbmFtZSksIGZpbGVuYW1lKQogICAgICAgICAgICB3ZWJtX3BhdGggPSBkZWZhdWx0X3N0b3JhZ2Uuc2F2ZShwYXRoLCBDb250ZW50RmlsZSh3ZWJtKSkKICAgICAgICAgICAgc2VsZi5fd2VibV9vYmoudmlkZW8gPSB3ZWJtX3BhdGgKCiAgICAgICAgaWYgdGh1bWI6CiAgICAgICAgICAgIGZpbGVuYW1lICs9ICcuanBnJwogICAgICAgICAgICBwYXRoID0gb3MucGF0aC5qb2luKAogICAgICAgICAgICAgICAgbW9kZWxzLmdldF9tZWRpYV9mb2xkZXIoc2VsZi5fd2VibV9vYmosIGZpbGVuYW1lKSwgZmlsZW5hbWUpCiAgICAgICAgICAgIHRodW1iX3BhdGggPSBkZWZhdWx0X3N0b3JhZ2Uuc2F2ZShwYXRoLCBDb250ZW50RmlsZSh0aHVtYikpCiAgICAgICAgICAgIHNlbGYuX3dlYm1fb2JqLnRodW1ibmFpbCA9IHRodW1iX3BhdGgKCiAgICAgICAgdHJ5OgogICAgICAgICAgICBzZWxmLl93ZWJtX29iai5zYXZlKCkKICAgICAgICBleGNlcHQgSW50ZWdyaXR5RXJyb3I6CiAgICAgICAgICAgIG1vZGVscy5XZWJtLmluY3JlYXNlX3JhdGluZyhzZWxmLl93ZWJtX29iai5tZDUpCgogICAgZGVmIHdvcmsoc2VsZiwgKmFyZ3MsICoqa3dhcmdzKToKICAgICAgICByZXR1cm4gc3VwZXIoKS53b3JrKHNlbGYuc2F2ZSkKCgppZiBfX25hbWVfXyA9PSAnX19tYWluX18nOgogICAgc2VjdGlvbnMgPSBbJ3ZnJywgJ2InLCAnYScsICdtb3YnXQogICAgd29ya2VyID0gc2NyYXBlci5NYWluV29ya2VyKHNlY3Rpb25zLCBkb3dubG9hZGVyPURvd25sb2FkVG9Nb2RlbCkKICAgIHdvcmtlci53b3JrKCkK