mirror of
https://github.com/KUlishevgeniy/c22712.git
synced 2026-09-23 23:50:21 +00:00
Merge remote-tracking branch 'origin/master'
This commit is contained in:
13danny13
@@ -0,0 +1,41 @@
|
||||
import psycopg2
|
||||
import wget
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
# Парсер и загрузчик в БД//Антипенко Дмитрий
|
||||
connection = psycopg2.connect(host='localhost', dbname='PythonDB', user='postgres', password='Q1w2e3r4')
|
||||
|
||||
cursor = connection.cursor()
|
||||
|
||||
creat_qwery = """ create table Parser
|
||||
(id serial primary key, page_name varchar(100), price varchar(10), priceDis varchar(30), mark varchar(10), scr varchar(100))"""
|
||||
|
||||
cursor.execute(creat_qwery)
|
||||
connection.commit()
|
||||
|
||||
driver = Service('D:\teach\Prog\chromedriver.exe')
|
||||
browser = webdriver.Chrome(service=driver)
|
||||
browser.get('https://amwine.ru/catalog/igristoe_vino_i_shampanskoe/igristoe_vino/')
|
||||
html_code = browser.page_source
|
||||
b_soup = BeautifulSoup(html_code, 'lxml')
|
||||
name = b_soup.find_all('a', class_="catalog-list-item__title js-product-detail-link")
|
||||
price = b_soup.find_all('span', class_="middle_price")
|
||||
priceDis = b_soup.find_all('span', class_="baseoldprice")
|
||||
mark = b_soup.find_all('span', class_="product-rating__rating")
|
||||
pictures = b_soup.find_all('div', class_="catalog-list-item__img-wrapper")
|
||||
|
||||
for i in range(15):
|
||||
url = 'https://amwine.ru'+pictures[i].find('a').find('img').attrs['data-src']
|
||||
filename = f"Programming\\23.03\img\{i}.jpg"
|
||||
wget.download(url, filename)
|
||||
ins_qwery = f"""insert into public.Parser(page_name, price, priceDis, mark, scr) values ('{name[i].text}', '{price[i].text}', '{priceDis[i].text}', '{mark[i].text}', '{filename}')"""
|
||||
cursor.execute(ins_qwery)
|
||||
connection.commit()
|
||||
|
||||
|
||||
|
||||
|
||||
cursor.close()
|
||||
|
||||
connection.close()
|
||||
@@ -8,6 +8,7 @@ browser.get('https://www.rftoday.ru/_vse_zagolovki2')
|
||||
html_text = browser.page_source
|
||||
soup = BeautifulSoup(html_text, 'lxml')
|
||||
infos = soup.find_all(attrs={"class": "title"})
|
||||
revs = soup.find_all(attrs={"class": "source"})
|
||||
#print(infos[0].text) - если нужно вывести только n-ый заголовок
|
||||
for info in infos:
|
||||
print(info.text)#Вывод всех заголовков новостей страницы 1
|
||||
for info, rev in zip(infos, revs):
|
||||
print(f"{info.text} ; Источник: {rev.text}")#Вывод всех заголовков новостей и их источников со страницы 1
|
||||
@@ -0,0 +1,34 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
import psycopg2
|
||||
import wget
|
||||
|
||||
browser = webdriver.Chrome(service=Service('C:\Desktop\exe\chromedriver.exe'))
|
||||
browser.get('https://flawery.ru/moscow/bouquets/color-blue/?min=0&max=max&sorting=discount')
|
||||
soup = BeautifulSoup(browser.page_source, "lxml")
|
||||
Name = soup.find_all(attrs={"class": "catalog_title"})
|
||||
Price = soup.find_all(attrs={"class": "catalog_price_now"})
|
||||
Discount = soup.find_all(attrs={"class": "catalog_sale"})
|
||||
Image = soup.find_all(attrs={"class": "catalog_item catalog_item_popup"})
|
||||
|
||||
connection = psycopg2.connect(host='localhost', dbname='FHWDB', user='postgres', password='Q1w2e3r4')
|
||||
cursor = connection.cursor()
|
||||
create_q = '''CREATE TABLE Parse
|
||||
(ID serial primary key, Name varchar(100), Price varchar(9), Discount varchar(10), src_of_picture varchar(50))'''
|
||||
cursor.execute(create_q)
|
||||
connection.commit()
|
||||
|
||||
for j in range(60):
|
||||
url = 'https://flawery.ru'+Image[j].find('a').find('picture').find('img').attrs['src']
|
||||
tempf = f"C:\\Users\\user\\PFDB{j}.jpg"
|
||||
wget.download(url, tempf)
|
||||
insert_query = f'''INSERT into public.Parse(Name, Price, Discount, src_of_picture) values ('{Name[j].text}', '{Price[j].text}', '{Discount[j].text}', '{tempf}') '''
|
||||
cursor.execute(insert_query)
|
||||
connection.commit()
|
||||
|
||||
cursor.execute("SELECT * from Parse")
|
||||
print(cursor.fetchall())
|
||||
|
||||
cursor.close()
|
||||
connection.close()
|
||||
@@ -0,0 +1,12 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
from time import sleep
|
||||
s = Service("C:\Driver\chromedriver.exe")
|
||||
browser = webdriver.Chrome(service=s)
|
||||
browser.get("https://mephi.ru/")
|
||||
sleep(10)
|
||||
html_text = browser.page_source
|
||||
soup = BeautifulSoup(html_text, "lxml")
|
||||
serv = soup.find_all("div", class_="views-field views-field-title menu-item menu-item-10702")
|
||||
print(serv[0].text)
|
||||
@@ -1,12 +1,35 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
import psycopg2
|
||||
import wget
|
||||
|
||||
s = Service("E:\data\chromedriver.exe")
|
||||
browser = webdriver.Chrome(service=s)
|
||||
browser.get("https://ria.ru/")
|
||||
html_text = browser.page_source
|
||||
soup = BeautifulSoup(html_text, 'lxml')
|
||||
news = soup.find_all('div', class_='cell cell-list')
|
||||
for new in news:
|
||||
print(new.text)
|
||||
browser = webdriver.Chrome(service=Service('E:\data\chromedriver.exe'))
|
||||
browser.get('https://8956food.ru/')
|
||||
soup = BeautifulSoup(browser.page_source, "lxml")
|
||||
|
||||
Name = soup.find_all(attrs={"class": "subtitle"})
|
||||
Price = soup.find_all(attrs={"class": "price"})
|
||||
Image = soup.find_all(attrs={"class": "holder-img"})
|
||||
|
||||
connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
|
||||
cursor = connection.cursor()
|
||||
create_q = '''CREATE TABLE Food
|
||||
(ID serial primary key, Name varchar(500), Price varchar(60), src varchar(100))'''
|
||||
cursor.execute(create_q)
|
||||
connection.commit()
|
||||
|
||||
#https://smartomato.ams3.cdn.digitaloceanspaces.com/uploads/media/photo/769221/dish_large__1.jpg
|
||||
for j in range(15):
|
||||
url = Image[j].find('img').attrs['src']
|
||||
tempf = f"C:\\Users\\Harutyun\\Desktop\\prog\\food{j}.jpg"
|
||||
wget.download(url, tempf)
|
||||
insert_query = f'''INSERT into public.Food(Name, Price, src) values ('{Name[j].text}', '{Price[j].text}', '{tempf}') '''
|
||||
cursor.execute(insert_query)
|
||||
connection.commit()
|
||||
|
||||
cursor.execute("SELECT * from Food")
|
||||
print(cursor.fetchall())
|
||||
|
||||
cursor.close()
|
||||
connection.close()
|
||||
@@ -1,4 +1,4 @@
|
||||
#made by dialuna (or just Diana Gasanova) <3
|
||||
# Diana Gasanova
|
||||
import requests
|
||||
from bs4 import BeautifulSoup as bs
|
||||
URL = "https://www.kinopoisk.ru/lists/movies/top-250-2020/"
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
from wget import download
|
||||
import psycopg2
|
||||
connection = psycopg2.connect(host='localhost', dbname='dbdata',
|
||||
user='postgres', password='Q1w2e3r4')
|
||||
cursor=connection.cursor()
|
||||
s = Service('C:\\Users\\Dasha\\Desktop\\Progect\\chromedriver.exe')
|
||||
browser = webdriver.Chrome(service=s)
|
||||
browser = webdriver.Chrome()
|
||||
url = 'https://www.tretyakovgallery.ru/'
|
||||
browser.get(url)
|
||||
html_text=browser.page_source
|
||||
soup = BeautifulSoup(html_text, 'lxml')
|
||||
products = soup.find_all(attrs={"class": "building-item"})
|
||||
for i, product in enumerate(products):
|
||||
link = product.find("img").attrs.get("src")
|
||||
tag = product.find("strong")
|
||||
name = product.find(attrs={"class": "building-item__title"})
|
||||
description = product.find(attrs={"class": "building-item__description"})
|
||||
if tag is not None:
|
||||
download(url + link, f"img\\{i}.jpg")
|
||||
cursor.execute(f"""INSERT INTO public.parsing(url, workhours, name, description, filename)
|
||||
VALUES
|
||||
( '{url + link}', '{tag.text}', '{name.text.strip()}', '{description.text.strip()}', 'img\\{i}.jpg')""")
|
||||
connection.commit()
|
||||
cursor.close()
|
||||
connection.close()
|
||||
@@ -0,0 +1,26 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
from wget import download
|
||||
import psycopg2
|
||||
connection = psycopg2.connect(host='localhost', dbname='dbdata',
|
||||
user='postgres', password='Q1w2e3r4')
|
||||
cursor=connection.cursor()
|
||||
s = Service('C:\\Users\\Dasha\\Desktop\\Progect\\chromedriver.exe')
|
||||
browser = webdriver.Chrome(service=s)
|
||||
url = 'https://www.tretyakovgallery.ru/'
|
||||
browser.get(url)
|
||||
html_text=browser.page_source
|
||||
soup = BeautifulSoup(html_text, 'lxml')
|
||||
products = soup.find_all(attrs={"class": "building-item"})
|
||||
for i, product in enumerate(products):
|
||||
link = product.find("img").attrs.get("src")
|
||||
tag = product.find("strong")
|
||||
if tag is not None:
|
||||
download(url + link, f"img\\{i}.jpg")
|
||||
cursor.execute(f"""INSERT INTO public.parsing(name, link)
|
||||
VALUES
|
||||
( '{tag.text}', '{url + link}')""")
|
||||
connection.commit()
|
||||
cursor.close()
|
||||
connection.close()
|
||||
@@ -0,0 +1,61 @@
|
||||
import psycopg2
|
||||
import wget
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
import time
|
||||
i = 1
|
||||
s = Service("С:\DATA\ChromeDriver\chromedriver.exe")
|
||||
browser = webdriver.Chrome(service=s)
|
||||
browser.get("https://www.mvideo.ru/noutbuki-planshety-komputery-8/noutbuki-118/f/tolko-v-nalichii=da?from=under_search")
|
||||
time.sleep(5)
|
||||
"""
|
||||
height = browser.execute_script("return document.body.scrollHeight")
|
||||
time.sleep(4)
|
||||
for i in range(10):
|
||||
browser.execute_script(f"window.scrollTo(0, {1000*i})")
|
||||
time.sleep(1)
|
||||
"""
|
||||
html_text = browser.page_source
|
||||
soup = BeautifulSoup(html_text, 'lxml')
|
||||
productst = soup.find_all('a', class_="product-title__text")
|
||||
pricest = soup.find_all('span', class_="price__main-value")
|
||||
features = soup.find_all('span', class_="product-feature-list__value")
|
||||
pictures = soup.find_all('div', class_="product-picture-container")
|
||||
connection=psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
|
||||
cursor = connection.cursor()
|
||||
""" insert = CREATE TABLE public.laptops(
|
||||
id serial primary key,
|
||||
product varchar(100),
|
||||
price varchar(15),
|
||||
diagonal varchar(5),
|
||||
resolution varchar(20),
|
||||
CPU varchar(50),
|
||||
RAM varchar(15),
|
||||
Graphics_Controller varchar(40),
|
||||
Volume varchar(25),
|
||||
src varchar(100)
|
||||
);
|
||||
"""
|
||||
insert = """TRUNCATE TABLE public.laptops; ALTER SEQUENCE laptops_id_seq RESTART WITH 1;"""
|
||||
cursor.execute(insert)
|
||||
connection.commit()
|
||||
for i in range(len(productst)):
|
||||
url = "https://"+pictures[i].find('img')['src']
|
||||
filename = f"C:\DATA\Images\{i+1}.jpg"
|
||||
u = []
|
||||
for j in range(10):
|
||||
u.append(features[10*i+j].text)
|
||||
wget.download(url, filename)
|
||||
t = pricest[i].text.replace("\xa0", " ")
|
||||
insert = f"""INSERT INTO public.laptops(
|
||||
product, price, diagonal, resolution, CPU, RAM, Graphics_Controller, Volume, src)
|
||||
VALUES
|
||||
('{productst[i].text.strip()}', '{t.strip()}', '{u[0][:u[0].find("/")]}', '{u[0][u[0].find("/")+1:]}',
|
||||
'{u[2]}', '{u[4]+" "+u[5]}', '{u[6]}', '{u[8]}', '{filename}');"""
|
||||
cursor.execute(insert)
|
||||
connection.commit()
|
||||
cursor.execute("select * from laptops")
|
||||
print("Результат", cursor.fetchall())
|
||||
cursor.close()
|
||||
connection.close()
|
||||
@@ -0,0 +1,40 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
import wget
|
||||
import psycopg2
|
||||
|
||||
connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
|
||||
cursor = connection.cursor()
|
||||
|
||||
s = Service(r"C:\\Documents\\data\\chromedriver.exe")
|
||||
browser = webdriver.Chrome(service=s)
|
||||
browser.get('https://www.citilink.ru/catalog/smartfony/')
|
||||
html_text = browser.page_source
|
||||
b_soup = BeautifulSoup(html_text, 'lxml')
|
||||
|
||||
insert_qwery = """create table phones\n
|
||||
(id serial primary key, name varchar(100), price varchar(100),delivery varchar(100), pictures varchar(100))"""
|
||||
|
||||
cursor.execute(insert_qwery)
|
||||
connection.commit()
|
||||
|
||||
phone_names = b_soup.find_all('a', class_="app-catalog-9gnskf e1259i3g0")
|
||||
phone_price = b_soup.find_all('span', class_="e1j9birj0 e106ikdt0 app-catalog-j8h82j e1gjr6xo0")
|
||||
phone_delivery = b_soup.find_all('div', class_="e16h9msl0 app-catalog-1fnzrtt e15ufesp0")
|
||||
pictures = b_soup.find_all('div', class_="app-catalog-zugobn e1o1y6qd0")
|
||||
|
||||
for i in range(15):
|
||||
url = 'https://www.citilink.ru/catalog/smartfony/'
|
||||
filename = f"C:\\Users\\mark6\\OneDrive\\Desktop\\БД\\img\\{i}.jpg"
|
||||
wget.download(url, filename)
|
||||
ins_qwery = f"""INSERT INTO public.phones(name, price, delivery, pictures)\n
|
||||
VALUES ('{phone_names[i].text}', '{phone_price[i].text}', '{phone_delivery[i].text}', '{filename}');"""
|
||||
cursor.execute(ins_qwery)
|
||||
connection.commit()
|
||||
|
||||
cursor.execute("Select * from phones")
|
||||
print(cursor.fetchall())
|
||||
|
||||
cursor.close()
|
||||
connection.close()
|
||||
@@ -0,0 +1,41 @@
|
||||
import psycopg2
|
||||
import wget
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
|
||||
|
||||
cursor = connection.cursor()
|
||||
|
||||
cr_qwery = """ create table Parser
|
||||
(id serial primary key, name varchar(100), sale varchar(30), price varchar(10), rate varchar(10), picture varchar(100))"""
|
||||
|
||||
#cursor.execute(cr_qwery)
|
||||
connection.commit()
|
||||
|
||||
driver = Service('D:\GOGdriver\chromedriver.exe')
|
||||
browser = webdriver.Chrome(service=driver)
|
||||
browser.get('https://www.perekrestok.ru/cat/promo/10')
|
||||
html_code = browser.page_source
|
||||
soup = BeautifulSoup(html_code, 'lxml')
|
||||
|
||||
name = soup.find_all('div', class_="product-card__title")
|
||||
sale = soup.find_all('div', class_="sc-kYrlTI ciSrDA product-card__badge")
|
||||
price = soup.find_all('div', class_="price-new")
|
||||
rate = soup.find_all('div', class_="rating-value")
|
||||
picture = soup.find_all('div', class_="product-card__image-wrapper")
|
||||
|
||||
for i in range(15):
|
||||
url = picture[i].find('img').attrs['src']
|
||||
filename = f"C:\\Users\\71332\PycharmProjects\pythonProject1\\venv\Pictures\{i}.jpg"
|
||||
wget.download(url, filename)
|
||||
ins_qwery = f"""insert into public.Parser(name, sale, price, rate, picture) values ('{name[i].text}', '{sale[i].text}', '{price[i].text}', '{rate[i].text}', '{filename}')"""
|
||||
cursor.execute(ins_qwery)
|
||||
connection.commit()
|
||||
|
||||
|
||||
|
||||
|
||||
cursor.close()
|
||||
|
||||
connection.close()
|
||||
@@ -0,0 +1,16 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
|
||||
driver = Service('D:\GOGdriver\chromedriver.exe')
|
||||
browser = webdriver.Chrome(service=driver)
|
||||
browser.get('https://www.kinopoisk.ru/lists/movies/popular-films/')
|
||||
html_code = browser.page_source
|
||||
soup = BeautifulSoup(html_code, 'lxml')
|
||||
name = soup.find_all('div', class_="desktop-list-main-info_secondaryTitleSlot__mc0mI")
|
||||
|
||||
print(soup)
|
||||
print(name)
|
||||
|
||||
for i in name:
|
||||
print(i.text)
|
||||
@@ -0,0 +1,39 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
import psycopg2
|
||||
import wget
|
||||
|
||||
connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
|
||||
cursor = connection.cursor()
|
||||
|
||||
crtable = """ create table Notebooks
|
||||
(id serial primary key, name varchar(500), description varchar(500), code varchar(200), price varchar(100),
|
||||
picture varchar(500)) """
|
||||
|
||||
cursor.execute(crtable)
|
||||
connection.commit()
|
||||
|
||||
s = Service('C:\\Users\\Yekaterina\\Downloads\\chromedriver_win32\\chromedriver.exe')
|
||||
browser = webdriver.Chrome(service=s)
|
||||
browser.get('https://www.citilink.ru/catalog/noutbuki/')
|
||||
html_text = browser.page_source
|
||||
soup = BeautifulSoup(html_text, 'lxml')
|
||||
|
||||
name = soup.find_all('div', class_='app-catalog-1tp0ino e1an64qs0')
|
||||
description = soup.find_all('div', class_='app-catalog-1o4umte eevw8x70')
|
||||
code = soup.find_all('div', class_= 'app-catalog-0 e1dsj6g20')
|
||||
price = soup.find_all('div', class_= 'app-catalog-0 e1dsj6g20')
|
||||
picture = soup.find_all('div', class_='app-catalog-0 e1jarwcz0')
|
||||
|
||||
for i in range(10):
|
||||
url = picture[i].find('img').attrs['src']
|
||||
file = f"C:\\Users\\Yekaterina\\Desktop\\baza\\pic{i}.JPG"
|
||||
wget.download(url, file)
|
||||
ins_qwery = f"""insert into public.Notebooks(name, description, code, price, picture)
|
||||
values ('{name[i].text}', '{description[i].text}', '{code[i].text}', '{price[i].text}','{file}')"""
|
||||
cursor.execute(ins_qwery)
|
||||
connection.commit()
|
||||
|
||||
cursor.close()
|
||||
connection.close()
|
||||
@@ -1,40 +1,31 @@
|
||||
# установи python, pip
|
||||
|
||||
# выполни эту команду: pip install selenium bs4 lxml
|
||||
# если ты используешь pycharm, то надо библиотеки ставить не в консоли(терминале), а в самом pycharm
|
||||
# для этого нажми "view" > "tool windows" > "python packages"
|
||||
# внизу в окне поиска ищи нужные библиотеки
|
||||
# когда найдешь, нажимай на нее, справа будет кнопка "install package", устанавливай, потом перезапусти pycharm
|
||||
|
||||
# найди версию своего браузера
|
||||
# напиши в гугле "скачать драйвер для <название и версия твоего браузера> selenium"
|
||||
# теперь все готово для запуска этого
|
||||
# помните, что сам сайт(который вы хотите парсить) может вас блокировать или заставлять проходить capture, из-за чего этот код ничего не выведет
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
# импортируем всё нужное
|
||||
from wget import download
|
||||
import psycopg2 as psyc
|
||||
|
||||
s = Service("путь_до_драйвера") # в ковычках указываем полный путь до скаченного ранее драйвера
|
||||
# если ты на винде, то вместо знака "\" пиши "\\"
|
||||
|
||||
brow = webdriver.Chrome(service=s) # как будто создаем виртуальный браузер
|
||||
brow = webdriver.Chrome()
|
||||
url = "https://ikey.ru/"
|
||||
brow.get(url)
|
||||
|
||||
brow.get("https://www.revshells.com/") # получаем html код сайта и другую информацию(она нам не нужна сейчас)
|
||||
html = brow.page_source
|
||||
|
||||
html = brow.page_source # копируем html код в переменную
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
|
||||
soup = BeautifulSoup(html, "lxml") # создаем специальный парсер
|
||||
products = soup.find_all(attrs={"class": "product"})
|
||||
|
||||
buttons = soup.find_all(attrs={"class": "list-group-item list-group-item-action"})
|
||||
# получаем список всех html тегов, в которых есть атрибут "class", равный: "list-group-item list-group-item-action"
|
||||
|
||||
for button in buttons:
|
||||
# выводим текст каждого тега
|
||||
print(button.text)
|
||||
|
||||
# от сердца и почек
|
||||
# дарю вам питончик
|
||||
# made by perfecto
|
||||
with psyc.connect(dbname="db_for_parse", user="perfecto") as conn:
|
||||
with conn.cursor() as cursor:
|
||||
for i, product in enumerate(products):
|
||||
img_tag = product.find(attrs={"class": "imagef"}).find("img")
|
||||
title = img_tag.attrs.get("alt")
|
||||
image = url + img_tag.attrs.get("src")
|
||||
price = product.find(attrs={"class": "product-price"}).text
|
||||
desc = product.find(attrs={"class": "product-desc"}).text
|
||||
download(url + img_tag.attrs.get("src"), f"images/{i}.jpg")
|
||||
|
||||
cursor.execute(f"""insert into images(link, name, price, description, file)
|
||||
values ('{image}', '{title}', '{price}', '{desc}', 'images/{i}.jpg')""")
|
||||
|
||||
conn.commit()
|
||||
|
||||
@@ -1,12 +1,40 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
import psycopg2
|
||||
import wget
|
||||
import time
|
||||
|
||||
connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
|
||||
cursor = connection.cursor()
|
||||
|
||||
s = Service('D:\Games\data\chromedriver.exe')
|
||||
browser = webdriver.Chrome(service=s)
|
||||
browser.get('https://www.volkswagen.ru/polo/')
|
||||
html_text = browser.page_source
|
||||
time.sleep(2)
|
||||
soup = BeautifulSoup(html_text, 'lxml')
|
||||
|
||||
creat_table = """ create table Cars_volks
|
||||
(id serial primary key, car_name varchar(20),
|
||||
price varchar(15), adress varchar(300),
|
||||
scr varchar(150)) """
|
||||
#cursor.execute(creat_table)
|
||||
connection.commit()
|
||||
|
||||
car_names = soup.find_all('div', class_='avn001-2_name')
|
||||
prices = soup.find_all('div', class_='avn001-2_price-container')
|
||||
for car_name, price in zip(car_names, prices):
|
||||
print(f"Название машины:{car_name.text} | Цена: {price.text} рублей")
|
||||
adresses = soup.find_all('div', class_='avn001-2_dealer-link__text')
|
||||
pictures = soup.find_all('div', class_='avn001-2_image image__container')
|
||||
|
||||
for i in range(len(car_names)):
|
||||
url = pictures[i].find('img')['src']
|
||||
filename = f"C:\\Users\\Пользователь\\OneDrive\\Рабочий стол\\pycharm\\c22712\\Задания\\task1\\nikitin\\img\{i}.jpg"
|
||||
wget.download(url, filename)
|
||||
insert_qwery = f"""INSERT INTO public.Cars_volks(car_name, price, adress, scr)
|
||||
VALUES ('{car_names[i].text}', '{prices[i].text}', '{adresses[i].text}', '{filename}')""";
|
||||
#cursor.execute(insert_qwery)
|
||||
connection.commit()
|
||||
|
||||
cursor.close()
|
||||
connection.close()
|
||||
@@ -0,0 +1,44 @@
|
||||
import psycopg2
|
||||
import wget
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
connection = psycopg2.connect(host='localhost', dbname='postgres',
|
||||
user='postgres', password='Q1w2e3r4')
|
||||
|
||||
cursor = connection.cursor()
|
||||
|
||||
cr_qwery = """ create table avito
|
||||
(id serial primary key,
|
||||
name varchar(60),
|
||||
price varchar(15),
|
||||
picture varchar(100))"""
|
||||
|
||||
cursor.execute(cr_qwery)
|
||||
connection.commit()
|
||||
|
||||
driver = Service('C:\Desktop\exe\chromedriver.exe')
|
||||
browser = webdriver.Chrome(service=driver)
|
||||
browser.get('https://www.hellride.ru/catalog/zapchasti-dlya-tryukovyh-samokatov/deki')
|
||||
html_code = browser.page_source
|
||||
soup = BeautifulSoup(html_code, 'lxml')
|
||||
|
||||
name = soup.find_all('div', class_='product-card__title')
|
||||
price = soup.find_all('div', class_='product-card__price')
|
||||
picture = soup.find_all('div', class_='product-slider__slide-img swiper-lazy swiper-lazy-loaded')
|
||||
|
||||
for i in range(len(name)):
|
||||
url = picture[i].find('img').attrs['src']
|
||||
filename = f"C:\\Users\\Михаил\PycharmProjects\pythonProject8\\venv\Pictures\{i}.jpg"
|
||||
wget.download(url, filename)
|
||||
ins_qwery = f"""insert into public.Parser(name, price, picture)
|
||||
values ('{name[i].text}', '{price[i].text}', '{filename}')"""
|
||||
cursor.execute(ins_qwery)
|
||||
connection.commit()
|
||||
|
||||
|
||||
|
||||
|
||||
cursor.close()
|
||||
|
||||
connection.close()
|
||||
@@ -1 +1 @@
|
||||
def print_hi(' hello')
|
||||
абоба
|
||||
@@ -0,0 +1,14 @@
|
||||
from bs4 import BeautifulSoup
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.chrome.service import Service
|
||||
|
||||
s = Service('C:\Desktop\exe\chromedriver.exe')
|
||||
browser = webdriver.Chrome(service=s)
|
||||
browser.get('https://www.hellride.ru/catalog/zapchasti-dlya-tryukovyh-samokatov/deki')
|
||||
html_text = browser.page_source
|
||||
soup = BeautifulSoup(html_text, 'lxml')
|
||||
prices = soup.find_all(attrs={"class": "product-card__price"})
|
||||
titles = soup.find_all(attrs={"class": "product-card__title"})
|
||||
|
||||
for price, title in zip(prices, titles):
|
||||
print(f" название {title.text} ; цена: {price.text}")
|
||||
Reference in New Issue
Block a user