From 7096a60830ff8d223e3bbf74c60895d116efb6aa Mon Sep 17 00:00:00 2001 From: eauskova <124862140+eauskova@users.noreply.github.com> Date: Wed, 29 Mar 2023 17:40:52 +0300 Subject: [PATCH 01/58] Parcing --- Задания/task1/Uskova/DataBaze.py | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 Задания/task1/Uskova/DataBaze.py diff --git a/Задания/task1/Uskova/DataBaze.py b/Задания/task1/Uskova/DataBaze.py new file mode 100644 index 0000000..e2bc77e --- /dev/null +++ b/Задания/task1/Uskova/DataBaze.py @@ -0,0 +1,39 @@ +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +import psycopg2 +import wget + +connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4') +cursor = connection.cursor() + +crtable = """ create table Notebooks + (id serial primary key, name varchar(500), description varchar(500), code varchar(200), price varchar(100), + picture varchar(500)) """ + +cursor.execute(crtable) +connection.commit() + +s = Service('C:\\Users\\Yekaterina\\Downloads\\chromedriver_win32\\chromedriver.exe') +browser = webdriver.Chrome(service=s) +browser.get('https://www.citilink.ru/catalog/noutbuki/') +html_text = browser.page_source +soup = BeautifulSoup(html_text, 'lxml') + +name = soup.find_all('div', class_='app-catalog-1tp0ino e1an64qs0') +description = soup.find_all('div', class_='app-catalog-1o4umte eevw8x70') +code = soup.find_all('div', class_= 'app-catalog-0 e1dsj6g20') +price = soup.find_all('div', class_= 'app-catalog-0 e1dsj6g20') +picture = soup.find_all('div', class_='app-catalog-0 e1jarwcz0') + +for i in range(2): + url = picture[i].find('img').attrs['src'] + file = f"C:\\Users\\Yekaterina\\Desktop\\baza\\pic{i}.JPG" + wget.download(url, file) + ins_qwery = f"""insert into public.Notebooks(name, description, code, price, picture) + values ('{name[i].text}', '{description[i].text}', '{code[i].text}', '{price[i].text}','{file}')""" + cursor.execute(ins_qwery) + connection.commit() + +cursor.close() +connection.close() \ No newline at end of file From d9014da5ab806b659d229bcd358ea4ae3e964a3c Mon Sep 17 00:00:00 2001 From: eauskova <124862140+eauskova@users.noreply.github.com> Date: Wed, 29 Mar 2023 17:46:57 +0300 Subject: [PATCH 02/58] Update DataBaze.py --- Задания/task1/Uskova/DataBaze.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Задания/task1/Uskova/DataBaze.py b/Задания/task1/Uskova/DataBaze.py index e2bc77e..b3f2f60 100644 --- a/Задания/task1/Uskova/DataBaze.py +++ b/Задания/task1/Uskova/DataBaze.py @@ -26,7 +26,7 @@ code = soup.find_all('div', class_= 'app-catalog-0 e1dsj6g20') price = soup.find_all('div', class_= 'app-catalog-0 e1dsj6g20') picture = soup.find_all('div', class_='app-catalog-0 e1jarwcz0') -for i in range(2): +for i in range(10): url = picture[i].find('img').attrs['src'] file = f"C:\\Users\\Yekaterina\\Desktop\\baza\\pic{i}.JPG" wget.download(url, file) @@ -36,4 +36,4 @@ for i in range(2): connection.commit() cursor.close() -connection.close() \ No newline at end of file +connection.close() From 3701d8af37121b9feace9e783130a1f6d108864a Mon Sep 17 00:00:00 2001 From: nikitin712 Date: Wed, 29 Mar 2023 18:40:10 +0300 Subject: [PATCH 03/58] =?UTF-8?q?=D0=94=D0=BE=D0=BC=D0=B0=D1=88=D0=BD?= =?UTF-8?q?=D0=B5=D0=B5=20=D0=B7=D0=B0=D0=B4=D0=B0=D0=BD=D0=B8=D0=B5=201?= =?UTF-8?q?=20(=D0=9F=D0=B0=D1=80=D1=81=D0=B8=D0=BD=D0=B3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Задания/task1/nikitin/test2.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Задания/task1/nikitin/test2.py b/Задания/task1/nikitin/test2.py index 23b8205..ccb97a6 100644 --- a/Задания/task1/nikitin/test2.py +++ b/Задания/task1/nikitin/test2.py @@ -3,6 +3,7 @@ from selenium import webdriver from selenium.webdriver.chrome.service import Service import psycopg2 import wget +import time connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4') cursor = connection.cursor() @@ -11,6 +12,7 @@ s = Service('D:\Games\data\chromedriver.exe') browser = webdriver.Chrome(service=s) browser.get('https://www.volkswagen.ru/polo/') html_text = browser.page_source +time.sleep(2) soup = BeautifulSoup(html_text, 'lxml') creat_table = """ create table Cars_volks @@ -25,17 +27,13 @@ prices = soup.find_all('div', class_='avn001-2_price-container') adresses = soup.find_all('div', class_='avn001-2_dealer-link__text') pictures = soup.find_all('div', class_='avn001-2_image image__container') -#for car_name, adress, price in zip(car_names, adresses, prices): -# print(f"Название машины:{car_name.text} | Адрес диллера: {adress.text} | Цена: {price.text} рублей") - for i in range(len(car_names)): - url = pictures[i].find('img').attrs['src'] - filename = f"c22712\\Задания\\task1\\nikitin\\img\\{i}.jpg" -# wget.download(url, filename) представим что картинки скачались:) - + url = pictures[i].find('img')['src'] + filename = f"C:\\Users\\Пользователь\\OneDrive\\Рабочий стол\\pycharm\\c22712\\Задания\\task1\\nikitin\\img\{i}.jpg" + wget.download(url, filename) insert_qwery = f"""INSERT INTO public.Cars_volks(car_name, price, adress, scr) VALUES ('{car_names[i].text}', '{prices[i].text}', '{adresses[i].text}', '{filename}')"""; - cursor.execute(insert_qwery) + #cursor.execute(insert_qwery) connection.commit() cursor.close() From 21ad5c1cbe929bf604a9771b87358afaf9dd8e24 Mon Sep 17 00:00:00 2001 From: Axeten <124861675+Axeten@users.noreply.github.com> Date: Wed, 29 Mar 2023 21:41:31 +0300 Subject: [PATCH 04/58] Create bd.py --- Задания/task1/romanov/bd.py | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 Задания/task1/romanov/bd.py diff --git a/Задания/task1/romanov/bd.py b/Задания/task1/romanov/bd.py new file mode 100644 index 0000000..ac66be0 --- /dev/null +++ b/Задания/task1/romanov/bd.py @@ -0,0 +1,44 @@ +import psycopg2 +import wget +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +connection = psycopg2.connect(host='localhost', dbname='postgres', + user='postgres', password='Q1w2e3r4') + +cursor = connection.cursor() + +cr_qwery = """ create table avito + (id serial primary key, + name varchar(60), + price varchar(15), + picture varchar(100))""" + +cursor.execute(cr_qwery) +connection.commit() + +driver = Service('C:\Desktop\exe\chromedriver.exe') +browser = webdriver.Chrome(service=driver) +browser.get('https://www.hellride.ru/catalog/zapchasti-dlya-tryukovyh-samokatov/deki') +html_code = browser.page_source +soup = BeautifulSoup(html_code, 'lxml') + +name = soup.find_all('div', class_='product-card__title') +price = soup.find_all('div', class_='product-card__price') +picture = soup.find_all('div', class_='product-slider__slide-img swiper-lazy swiper-lazy-loaded') + +for i in range(len(name)): + url = picture[i].find('img').attrs['src'] + filename = f"C:\\Users\\Михаил\PycharmProjects\pythonProject8\\venv\Pictures\{i}.jpg" + wget.download(url, filename) + ins_qwery = f"""insert into public.Parser(name, price, picture) + values ('{name[i].text}', '{price[i].text}', '{filename}')""" + cursor.execute(ins_qwery) + connection.commit() + + + + +cursor.close() + +connection.close() From f0243e9b713e5e90d4146aa26e5bfab3fad021e3 Mon Sep 17 00:00:00 2001 From: 13danny13 Date: Wed, 29 Mar 2023 22:33:57 +0300 Subject: [PATCH 05/58] database --- Задания/task1/Kuznetsov/db.py | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 Задания/task1/Kuznetsov/db.py diff --git a/Задания/task1/Kuznetsov/db.py b/Задания/task1/Kuznetsov/db.py new file mode 100644 index 0000000..5a41c3c --- /dev/null +++ b/Задания/task1/Kuznetsov/db.py @@ -0,0 +1,41 @@ +import psycopg2 +import wget +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.service import Service + +connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4') + +cursor = connection.cursor() + +creat_qwery = """ create table Parser + (id serial primary key, product_name varchar(100), description varchar(1000), price varchar(10), priceDis varchar(30), scr varchar(100))""" + +cursor.execute(creat_qwery) +connection.commit() + +driver = Service('E:\\Downloads\\chromedriver_win32.zip\\chromedriver.exe') +browser = webdriver.Chrome(service=driver) +browser.get('https://trial-sport.ru/gds.php?s=51516&c1=1070639&c2=1070640') +html_code = browser.page_source +soup = BeautifulSoup(html_code, 'lxml') +product_name=soup.find_all('a', class_='title') +description=soup.find_all('span', class_='description') +price = soup.find_all('span', class_="price") +priceDis = soup.find_all('span', class_="discount") +pictures = soup.find_all('div', class_="img") + +for i in range(15): + url = 'https://trial-sport.ru/gds.php?s=51516&c1=1070639&c2=1070640'+pictures[i].find('a').find('img')['src'] + filename = f"E:\Downloads\parcing\img\{i}.jpg" + wget.download(url, filename) + ins_qwery = f"""insert into public.Parser(product_name, description, price, priceDis, scr) + VALUES ('{product_name[i].text}','{description[i].text}', '{price[i].text}', '{priceDis[i].text}', '{filename}')""" + connection.commit() + + + + +cursor.close() + +connection.close() \ No newline at end of file From 366f4b28e613d28f570080db8eb6258a78120dc3 Mon Sep 17 00:00:00 2001 From: Axeten <124861675+Axeten@users.noreply.github.com> Date: Wed, 29 Mar 2023 22:54:31 +0300 Subject: [PATCH 06/58] Update and rename bd.py to bd --- Задания/task1/romanov/{bd.py => bd} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename Задания/task1/romanov/{bd.py => bd} (86%) diff --git a/Задания/task1/romanov/bd.py b/Задания/task1/romanov/bd similarity index 86% rename from Задания/task1/romanov/bd.py rename to Задания/task1/romanov/bd index ac66be0..fb0c1b1 100644 --- a/Задания/task1/romanov/bd.py +++ b/Задания/task1/romanov/bd @@ -23,9 +23,9 @@ browser.get('https://www.hellride.ru/catalog/zapchasti-dlya-tryukovyh-samokatov/ html_code = browser.page_source soup = BeautifulSoup(html_code, 'lxml') -name = soup.find_all('div', class_='product-card__title') -price = soup.find_all('div', class_='product-card__price') -picture = soup.find_all('div', class_='product-slider__slide-img swiper-lazy swiper-lazy-loaded') +name = soup.find_all('span', class_='product-card__title') +price = soup.find_all('span', class_='product-card__price') +picture = soup.find_all('img', class_='product-slider__slide-img swiper-lazy swiper-lazy-loaded') for i in range(len(name)): url = picture[i].find('img').attrs['src'] From fbb156c7dc7d74ba77a180ba0ca91dfa1d5bbb2b Mon Sep 17 00:00:00 2001 From: 13danny13 Date: Wed, 29 Mar 2023 22:57:31 +0300 Subject: [PATCH 07/58] database --- Задания/task1/Kuznetsov/db.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Задания/task1/Kuznetsov/db.py b/Задания/task1/Kuznetsov/db.py index 5a41c3c..28c1c54 100644 --- a/Задания/task1/Kuznetsov/db.py +++ b/Задания/task1/Kuznetsov/db.py @@ -8,14 +8,14 @@ connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres' cursor = connection.cursor() -creat_qwery = """ create table Parser +creat_qwery = """ create table Parsing (id serial primary key, product_name varchar(100), description varchar(1000), price varchar(10), priceDis varchar(30), scr varchar(100))""" cursor.execute(creat_qwery) connection.commit() -driver = Service('E:\\Downloads\\chromedriver_win32.zip\\chromedriver.exe') -browser = webdriver.Chrome(service=driver) +s = Service('E:\\Downloads\\chromedriver_win32.zip\\chromedriver.exe') +browser = webdriver.Chrome(service=s) browser.get('https://trial-sport.ru/gds.php?s=51516&c1=1070639&c2=1070640') html_code = browser.page_source soup = BeautifulSoup(html_code, 'lxml') @@ -25,7 +25,7 @@ price = soup.find_all('span', class_="price") priceDis = soup.find_all('span', class_="discount") pictures = soup.find_all('div', class_="img") -for i in range(15): +for i in range(20): url = 'https://trial-sport.ru/gds.php?s=51516&c1=1070639&c2=1070640'+pictures[i].find('a').find('img')['src'] filename = f"E:\Downloads\parcing\img\{i}.jpg" wget.download(url, filename) From 709fc0d694ffa25e1c40e8a108fdcc95d72069b3 Mon Sep 17 00:00:00 2001 From: Axeten <124861675+Axeten@users.noreply.github.com> Date: Wed, 29 Mar 2023 23:16:07 +0300 Subject: [PATCH 08/58] Update bd --- Задания/task1/romanov/bd | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Задания/task1/romanov/bd b/Задания/task1/romanov/bd index fb0c1b1..7f5d9c1 100644 --- a/Задания/task1/romanov/bd +++ b/Задания/task1/romanov/bd @@ -8,7 +8,7 @@ connection = psycopg2.connect(host='localhost', dbname='postgres', cursor = connection.cursor() -cr_qwery = """ create table avito +cr_qwery = """ create table parser (id serial primary key, name varchar(60), price varchar(15), @@ -17,8 +17,8 @@ cr_qwery = """ create table avito cursor.execute(cr_qwery) connection.commit() -driver = Service('C:\Desktop\exe\chromedriver.exe') -browser = webdriver.Chrome(service=driver) +s = Service('C:\Desktop\exe\chromedriver.exe') +browser = webdriver.Chrome(service=s) browser.get('https://www.hellride.ru/catalog/zapchasti-dlya-tryukovyh-samokatov/deki') html_code = browser.page_source soup = BeautifulSoup(html_code, 'lxml') @@ -26,9 +26,10 @@ soup = BeautifulSoup(html_code, 'lxml') name = soup.find_all('span', class_='product-card__title') price = soup.find_all('span', class_='product-card__price') picture = soup.find_all('img', class_='product-slider__slide-img swiper-lazy swiper-lazy-loaded') - +#for prices, names in zip(price, name,): + #print(f" название {names.text} ; цена: {prices.text} ") for i in range(len(name)): - url = picture[i].find('img').attrs['src'] + url = picture[i].attrs['src'] filename = f"C:\\Users\\Михаил\PycharmProjects\pythonProject8\\venv\Pictures\{i}.jpg" wget.download(url, filename) ins_qwery = f"""insert into public.Parser(name, price, picture) From 56d6a99a7136b36c6f74953e4cc2410e8da482ac Mon Sep 17 00:00:00 2001 From: inweems Date: Thu, 30 Mar 2023 23:06:11 +0300 Subject: [PATCH 09/58] =?UTF-8?q?=D0=94=D0=B7=20=D0=BF=D0=B0=D1=80=D1=81?= =?UTF-8?q?=D0=B8=D0=BD=D0=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/workspace.xml | 14 ++++++------ Задания/task1/Prokhorova/dz.py | 41 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 Задания/task1/Prokhorova/dz.py diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 28f4cba..858e631 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -4,7 +4,7 @@