From 203664e1cafff5f9d953c05c8fd9e3b41ede9cde Mon Sep 17 00:00:00 2001 From: paov Date: Tue, 11 Oct 2022 16:20:19 +0300 Subject: [PATCH] second --- .idea/deployment.xml | 1 + .idea/vcs.xml | 6 ++ choose/admin.py | 9 ++- choose/migrations/0004_pic_alter_tea_pic.py | 26 ++++++ choose/migrations/0005_teatype_one_item.py | 19 +++++ choose/migrations/0006_teatype_preferred.py | 19 +++++ choose/models.py | 17 +++- choose/utils.py | 2 + choose/views.py | 22 ++++- sync.py | 89 +++++++++++++++++++-- tea/settings.py | 3 + tea/urls.py | 7 ++ 12 files changed, 208 insertions(+), 12 deletions(-) create mode 100644 .idea/vcs.xml create mode 100644 choose/migrations/0004_pic_alter_tea_pic.py create mode 100644 choose/migrations/0005_teatype_one_item.py create mode 100644 choose/migrations/0006_teatype_preferred.py create mode 100644 choose/utils.py diff --git a/.idea/deployment.xml b/.idea/deployment.xml index b3f3305..c2136ee 100644 --- a/.idea/deployment.xml +++ b/.idea/deployment.xml @@ -14,6 +14,7 @@ + diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/choose/admin.py b/choose/admin.py index 917c048..7252e33 100644 --- a/choose/admin.py +++ b/choose/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin # Register your models here. -from choose.models import Tea, TeaType, TeaCategory +from choose.models import Tea, TeaType, TeaCategory, Pic class TeaCategoryAdmin(admin.ModelAdmin): @@ -9,13 +9,18 @@ class TeaCategoryAdmin(admin.ModelAdmin): class TeaTypeAdmin(admin.ModelAdmin): - list_display = ('name', 'shop_name', 'category') + list_display = ('name', 'shop_name', 'category', 'one_item') class TeaAdmin(admin.ModelAdmin): list_display = ('name', 'price', 'pic', 'type') +class PicAdmin(admin.ModelAdmin): + list_display = ('href',) + + admin.site.register(Tea, TeaAdmin) admin.site.register(TeaType, TeaTypeAdmin) admin.site.register(TeaCategory, TeaCategoryAdmin) +admin.site.register(Pic, PicAdmin) diff --git a/choose/migrations/0004_pic_alter_tea_pic.py b/choose/migrations/0004_pic_alter_tea_pic.py new file mode 100644 index 0000000..5b68c4d --- /dev/null +++ b/choose/migrations/0004_pic_alter_tea_pic.py @@ -0,0 +1,26 @@ +# Generated by Django 4.0.8 on 2022-10-11 08:10 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('choose', '0003_teacategory_teatype_category'), + ] + + operations = [ + migrations.CreateModel( + name='Pic', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('href', models.CharField(max_length=512)), + ], + ), + migrations.AlterField( + model_name='tea', + name='pic', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='choose.pic'), + ), + ] diff --git a/choose/migrations/0005_teatype_one_item.py b/choose/migrations/0005_teatype_one_item.py new file mode 100644 index 0000000..b729313 --- /dev/null +++ b/choose/migrations/0005_teatype_one_item.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0.8 on 2022-10-11 09:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('choose', '0004_pic_alter_tea_pic'), + ] + + operations = [ + migrations.AddField( + model_name='teatype', + name='one_item', + field=models.BooleanField(default=False), + preserve_default=False, + ), + ] diff --git a/choose/migrations/0006_teatype_preferred.py b/choose/migrations/0006_teatype_preferred.py new file mode 100644 index 0000000..23c0f7d --- /dev/null +++ b/choose/migrations/0006_teatype_preferred.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0.8 on 2022-10-11 11:22 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('choose', '0005_teatype_one_item'), + ] + + operations = [ + migrations.AddField( + model_name='teatype', + name='preferred', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tea_type', to='choose.tea'), + ), + ] diff --git a/choose/models.py b/choose/models.py index 3150dc4..fee9351 100644 --- a/choose/models.py +++ b/choose/models.py @@ -19,15 +19,30 @@ class TeaType(models.Model): on_delete=models.CASCADE, blank=True, null=True ) + one_item = models.BooleanField() + preferred = models.ForeignKey('Tea', + related_name='tea_type', + on_delete=models.SET_NULL, + blank=True, null=True + ) def __str__(self): return self.name +class Pic(models.Model): + href = models.CharField(max_length=512) + + def __str__(self): + return self.href + + class Tea(models.Model): name = models.CharField(max_length=255) price = models.IntegerField() - pic = models.CharField(max_length=255) + pic = models.ForeignKey(Pic, + blank=True, null=True, + on_delete=models.CASCADE) type = models.ForeignKey(TeaType, related_name='teas', on_delete=models.CASCADE) diff --git a/choose/utils.py b/choose/utils.py new file mode 100644 index 0000000..df4b77b --- /dev/null +++ b/choose/utils.py @@ -0,0 +1,2 @@ +def get_extension(href): + return href[href.rfind('.') + 1:] diff --git a/choose/views.py b/choose/views.py index 91ea44a..dda6584 100644 --- a/choose/views.py +++ b/choose/views.py @@ -1,3 +1,23 @@ -from django.shortcuts import render +from collections import OrderedDict +from django.http import JsonResponse +from django.shortcuts import render +from .models import Tea, TeaType, TeaCategory, Pic # Create your views here. +from .utils import get_extension + + +def list(request): + dic = OrderedDict() + for cat in TeaCategory.objects.all(): + dic[cat.name] = OrderedDict() + + for tt in TeaType.objects.all(): + if tt.preferred is not None: + dic[tt.category.name][tt.name] = OrderedDict() + dic[tt.category.name][tt.name]['preferred'] = tt.preferred.name + dic[tt.category.name][tt.name]['price'] = tt.preferred.price + dic[tt.category.name][tt.name]['pic'] = str(tt.preferred.pic.id) + '.' + get_extension( + tt.preferred.pic.href) + + return JsonResponse(dic, json_dumps_params={'ensure_ascii': False}) diff --git a/sync.py b/sync.py index 9db7b28..d7135ee 100644 --- a/sync.py +++ b/sync.py @@ -1,3 +1,5 @@ +import json + import django from django.conf import settings import requests @@ -5,6 +7,8 @@ from bs4 import BeautifulSoup from pathlib import Path +from choose.utils import get_extension + BASE_DIR = Path(__file__).resolve().parent settings.configure( DATABASES={ @@ -25,12 +29,12 @@ settings.configure( ) django.setup() -from choose.models import TeaType, Tea, TeaCategory +from choose.models import TeaType, Tea, TeaCategory, Pic db_cats = list(TeaCategory.objects.all()) categories = ['https://chainiisvet.ru/product-category/' + x.shop_name + '/' for x in db_cats] r = requests.get("https://chainiisvet.ru/") -soup = BeautifulSoup(r.content) +soup = BeautifulSoup(r.content, features="html.parser") ul = soup.find("ul", {"id": "menu-katalog"}) for item in ul.contents: if item.__class__.__name__ == 'Tag': @@ -38,13 +42,82 @@ for item in ul.contents: category = TeaCategory.objects.get(name=item.contents[0].contents[0]) for i in item.contents[2]: if i.__class__.__name__ == 'Tag': - type = i.contents[0].contents[0] + ttype = i.contents[0].contents[0] href = i.contents[0].attrs['href'] if i.contents[0].attrs['href'].startswith( 'https') else 'https://chainiisvet.ru' + i.contents[0].attrs['href'] - TeaType.objects.get_or_create(shop_name=href[href.rfind('/', 0, len(href)-1)+1:-1], - name=type, - category=category) + if 'shop' in href: + _, created = TeaType.objects.get_or_create( + shop_name=href[href[:href.rfind('/', 0, len(href) - 1)].rfind('/') + 1:-1], + name=ttype, + category=category, + one_item=True) + else: + _, created = TeaType.objects.get_or_create( + shop_name=href[href.rfind('/', 0, len(href) - 1) + 1:-1], + name=ttype, + category=category, + one_item=False) + print(created) print(category, end=':') - print(type, end=':') + print(ttype, end=':') print(href) -pass +for tea_type in list(TeaType.objects.all()): + if tea_type.one_item: + r = requests.get( + "https://chainiisvet.ru/shop/" + tea_type.shop_name + '/') + soup = BeautifulSoup(r.content, features="html.parser") + href = str(soup.find_all("img", {"class": "attachment-shop_single"})[0].attrs['src']) + pic, created = Pic.objects.get_or_create(href=href) + if created: + r = requests.get(href) + with open('media/' + str(pic.id) + '.' + get_extension(href), 'wb') as f: + f.write(r.content) + name = str(soup.find_all("h1", {"class": "product_title"})[0].contents[0]) + if len(soup.find_all("form", {"class": "variations_form cart"})): + price = \ + json.loads( + soup.find_all("form", {"class": "variations_form cart"})[0].attrs['data-product_variations'])[0][ + 'display_price'] * 100 + else: + price = int( + soup.find_all("div", {"class": 'entry-summary'})[0].contents[3].contents[0].contents[0].replace(".", + "").replace( + ",", "")) + tea = Tea.objects.filter(name=name, type=tea_type).first() + if tea is None: + Tea.objects.create(name=name, type=tea_type, price=price, pic=pic) + else: + if not tea.price == price: + tea.price = price + tea.save() + if not tea.pic == pic: + tea.pic = pic + else: + r = requests.get( + "https://chainiisvet.ru/product-category/" + tea_type.category.shop_name + '/' + tea_type.shop_name + '/') + soup = BeautifulSoup(r.content, features="html.parser") + ul = soup.find_all("ul", {"class": "products"})[0] + for item in ul.contents: + if item.__class__.__name__ == 'Tag': + contents = item.contents[1].contents + href = str(contents[1].attrs['data-src']) + pic, created = Pic.objects.get_or_create(href=href) + if created: + r = requests.get(href) + with open('media/' + str(pic.id) + '.' + get_extension(href), 'wb') as f: + f.write(r.content) + name = '' + for i in contents[2]: + if not i.__class__.__name__ == 'Tag': + name = name + str(i) + price = int(str(contents[6].contents[1].contents[0]).replace(".", "").replace(",", "")) + print(name) + tea = Tea.objects.filter(name=name, type=tea_type).first() + if tea is None: + Tea.objects.create(name=name, type=tea_type, price=price, pic=pic) + else: + if not tea.price == price: + tea.price = price + tea.save() + if not tea.pic == pic: + tea.pic = pic diff --git a/tea/settings.py b/tea/settings.py index 15e2fce..a299016 100644 --- a/tea/settings.py +++ b/tea/settings.py @@ -124,3 +124,6 @@ STATIC_URL = 'static/' # https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +MEDIA_ROOT = f'{BASE_DIR}/media' +MEDIA_URL = '/media/' diff --git a/tea/urls.py b/tea/urls.py index a2d0df6..0183f70 100644 --- a/tea/urls.py +++ b/tea/urls.py @@ -15,7 +15,14 @@ Including another URLconf """ from django.contrib import admin from django.urls import path +from django.conf import settings +from django.conf.urls.static import static + +import choose.views urlpatterns = [ path('admin/', admin.site.urls), + path('list', choose.views.list) ] +if settings.DEBUG: + urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)