Merge remote-tracking branch 'origin/master'

This commit is contained in:
nikitin712
2023-03-26 21:12:28 +03:00
21 changed files with 1304 additions and 53 deletions
+43 -3
View File
@@ -4,7 +4,9 @@
<option name="autoReloadType" value="SELECTIVE" />
</component>
<component name="ChangeListManager">
<list default="true" id="e5615680-c090-497d-a104-1337ed85f850" name="Changes" comment="parser" />
<list default="true" id="e5615680-c090-497d-a104-1337ed85f850" name="Changes" comment="parser">
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
@@ -33,7 +35,12 @@
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="RunManager">
<component name="PropertiesComponent"><![CDATA[{
"keyToString": {
"last_opened_file_path": "C:/Users/71332/PycharmProjects/c22712"
}
}]]></component>
<component name="RunManager" selected="Python.zaparsil">
<configuration name="main" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
<module name="22712" />
<option name="INTERPRETER_OPTIONS" value="" />
@@ -56,6 +63,32 @@
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<configuration name="zaparsil" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
<module name="22712" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/Задания/task1/Sukhanov" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/Задания/task1/Sukhanov/zaparsil.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
<recent_temporary>
<list>
<item itemvalue="Python.zaparsil" />
</list>
</recent_temporary>
</component>
<component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
<component name="TaskManager">
@@ -102,7 +135,14 @@
<option name="project" value="LOCAL" />
<updated>1676934066947</updated>
</task>
<option name="localTasksCounter" value="6" />
<task id="LOCAL-00006" summary="parser">
<created>1679839777822</created>
<option name="number" value="00006" />
<option name="presentableId" value="LOCAL-00006" />
<option name="project" value="LOCAL" />
<updated>1679839777822</updated>
</task>
<option name="localTasksCounter" value="7" />
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
+20 -13
View File
@@ -1,16 +1,23 @@
# This is a sample Python script.
from bs4 import BeautifulSoup
from selenium.webdriver import Chrome
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
import time
# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
# Selenium - библиотека для автоматизации действий веб браузера, скрапинга
# запускаем браузер
s = Service('C:\data\chrome\chromedriver.exe')
browser = webdriver.Chrome(service=s)
browser.get('https://www.wildberries.ru/catalog/muzhchinam/odezhda/futbolki-i-mayki?bid=3b73ce5a-8eba-45a1-b9b2-0723f7592eac#c145658286')
time.sleep (10)#задержка для ввода капчи
html_text = browser.page_source
soup = BeautifulSoup(html_text, 'lxml')
films=soup.find_all('div', class_='/games/the-elder-scrolls-v-skyrim/')
ratings = soup.find_all('div', class_='KnbCardMark_label__hg6Pg tv-series-mark KnbCardMark_isGreen__G_FLl')
for film, rating in zip(films, ratings):
print(f"Игра:{film.text} | Рейтинг:{rating.text}")
def print_hi(name):
# Use a breakpoint in the code line below to debug your script.
print(f'Hi, {name}') # Press Ctrl+F8 to toggle the breakpoint.
# Press the green button in the gutter to run the script.
if __name__ == '__main__':
print_hi('PyCharm')
# See PyCharm help at https://www.jetbrains.com/help/pycharm/
# Это пример парсинга. Вам необходимо спарсить 1 страницу каталога любого сайта на выбор.
#сайты не должны повторяться
# Спарсить необходимо только Заголовки и описание.
+41
View File
@@ -0,0 +1,41 @@
import psycopg2
import wget
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
# Парсер и загрузчик в БД//Антипенко Дмитрий
connection = psycopg2.connect(host='localhost', dbname='PythonDB', user='postgres', password='Q1w2e3r4')
cursor = connection.cursor()
creat_qwery = """ create table Parser
(id serial primary key, page_name varchar(100), price varchar(10), priceDis varchar(30), mark varchar(10), scr varchar(100))"""
cursor.execute(creat_qwery)
connection.commit()
driver = Service('D:\teach\Prog\chromedriver.exe')
browser = webdriver.Chrome(service=driver)
browser.get('https://amwine.ru/catalog/igristoe_vino_i_shampanskoe/igristoe_vino/')
html_code = browser.page_source
b_soup = BeautifulSoup(html_code, 'lxml')
name = b_soup.find_all('a', class_="catalog-list-item__title js-product-detail-link")
price = b_soup.find_all('span', class_="middle_price")
priceDis = b_soup.find_all('span', class_="baseoldprice")
mark = b_soup.find_all('span', class_="product-rating__rating")
pictures = b_soup.find_all('div', class_="catalog-list-item__img-wrapper")
for i in range(15):
url = 'https://amwine.ru'+pictures[i].find('a').find('img').attrs['data-src']
filename = f"Programming\\23.03\img\{i}.jpg"
wget.download(url, filename)
ins_qwery = f"""insert into public.Parser(page_name, price, priceDis, mark, scr) values ('{name[i].text}', '{price[i].text}', '{priceDis[i].text}', '{mark[i].text}', '{filename}')"""
cursor.execute(ins_qwery)
connection.commit()
cursor.close()
connection.close()
+23 -2
View File
@@ -1,3 +1,6 @@
#Вариант с пары
'''
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
@@ -10,9 +13,27 @@ html_text = browser.page_source
time.sleep(20)
soup = BeautifulSoup(html_text, 'lxml')
films = soup.find_all('div', class_='base-movie-main-info_mainInfo__ZL_u3')
'''
print(soup)
print(films)
'''
for film in films:
print(film.text)
'''
#Домашка
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
driver = Service('D:\teach\Prog\chromedriver.exe')
browser = webdriver.Chrome(service=driver)
browser.get('https://hmbrussia.ru/regional-office/')
html_code = browser.page_source
b_soup = BeautifulSoup(html_code, 'lxml')
name = b_soup.find_all('div', class_="ps-xl-3 ms-xl-3")
print(b_soup)
print(name)
for i in name:
print(i.text)
@@ -8,6 +8,7 @@ browser.get('https://www.rftoday.ru/_vse_zagolovki2')
html_text = browser.page_source
soup = BeautifulSoup(html_text, 'lxml')
infos = soup.find_all(attrs={"class": "title"})
revs = soup.find_all(attrs={"class": "source"})
#print(infos[0].text) - если нужно вывести только n-ый заголовок
for info in infos:
print(info.text)#Вывод всех заголовков новостей страницы 1
for info, rev in zip(infos, revs):
print(f"{info.text} ; Источник: {rev.text}")#Вывод всех заголовков новостей и их источников со страницы 1
+32
View File
@@ -0,0 +1,32 @@
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
import psycopg2
import wget
browser = webdriver.Chrome(service=Service('C:\Desktop\exe\chromedriver.exe'))
browser.get('https://flawery.ru/moscow/bouquets/event-yanvary25/?min=0&max=max&sorting=discount&filter_24h=on')
soup = BeautifulSoup(browser.page_source, "lxml")
Name = soup.find_all(attrs={"class": "catalog_title"})
Price = soup.find_all(attrs={"class": "catalog_price_now"})
Delivery_Time = soup.find_all(attrs={"class": "catalog_express"})
Discount = soup.find_all(attrs={"class": "catalog_sale"})
Image = soup.find_all(attrs={"class": "catalog_item catalog_item_popup"})
connection = psycopg2.connect(host='localhost', dbname='FHWDB', user='postgres', password='Q1w2e3r4')
cursor = connection.cursor()
create_q = '''CREATE TABLE Parse
(ID serial primary key, Name varchar(100), Price varchar(9), Delivery_Time varchar(25), Discount varchar(10), src varchar(110))'''
cursor.execute(create_q)
connection.commit()
for j in range(60):
url = 'https://flawery.ru'+Image[j].find('a').find('picture').find('img').attrs['src']
tempf = f"C:\\Users\\user\\Desktop\\FHWDB{j}.jpg"
wget.download(url, tempf)
insert_query = f'''INSERT into public.Parse(Name, Price, Delivery_Time, Discount, src) values ('{Name[j].text}', '{Price[j].text}', '{Delivery_Time[j].text}', '{Discount[j].text}', '{tempf}') '''
cursor.execute(insert_query)
connection.commit()
cursor.close()
connection.close()
+12
View File
@@ -0,0 +1,12 @@
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from time import sleep
s = Service("C:\Driver\chromedriver.exe")
browser = webdriver.Chrome(service=s)
browser.get("https://mephi.ru/")
sleep(10)
html_text = browser.page_source
soup = BeautifulSoup(html_text, "lxml")
serv = soup.find_all("div", class_="views-field views-field-title menu-item menu-item-10702")
print(serv[0].text)
+1 -1
View File
@@ -1,4 +1,4 @@
#made by dialuna (or just Diana Gasanova) <3
# Diana Gasanova
import requests
from bs4 import BeautifulSoup as bs
URL = "https://www.kinopoisk.ru/lists/movies/top-250-2020/"
@@ -0,0 +1,26 @@
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from wget import download
import psycopg2
connection = psycopg2.connect(host='localhost', dbname='dbdata',
user='postgres', password='Q1w2e3r4')
cursor=connection.cursor()
s = Service('C:\\Users\\Dasha\\Desktop\\Progect\\chromedriver.exe')
browser = webdriver.Chrome(service=s)
url = 'https://www.tretyakovgallery.ru/'
browser.get(url)
html_text=browser.page_source
soup = BeautifulSoup(html_text, 'lxml')
products = soup.find_all(attrs={"class": "building-item"})
for i, product in enumerate(products):
link = product.find("img").attrs.get("src")
tag = product.find("strong")
if tag is not None:
download(url + link, f"img\\{i}.jpg")
cursor.execute(f"""INSERT INTO public.parsing(name, link)
VALUES
( '{tag.text}', '{url + link}')""")
connection.commit()
cursor.close()
connection.close()
+15
View File
@@ -0,0 +1,15 @@
from bs4 import BeautifulSoup
from selenium.webdriver import Chrome
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
import time
s = Service('E:\\Downloads\\chromedriver_win32.zip\\chromedriver.exe')
browser = webdriver.Chrome(service=s)
browser.get('https://trial-sport.ru/gds.php?s=51516&c1=1070639&c2=1070640')
html_text = browser.page_source
soup = BeautifulSoup(html_text, 'lxml')
product_name=soup.find_all('a', class_='title')
description=soup.find_all('span', class_='description')
for product_name, description in zip(product_name, description):
print(f"Продукт: {product_name.text} \nОписание: {description.text}")
print(" ")
+14 -1
View File
@@ -1 +1,14 @@
test
from bs4 import BeautifulSoup
from selenium.webdriver import Chrome
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
import time
s = Service('E:\\Downloads\\chromedriver_win32.zip\\chromedriver.exe')
browser = webdriver.Chrome(service=s)
browser.get('https://trial-sport.ru/gds.php?s=51516&c1=1070639&c2=1070640')
html_text = browser.page_source
soup = BeautifulSoup(html_text, 'lxml')
bike_name=soup.find_all('a', class_='title')
print (bike_name[0].text)
description=soup.find_all('span', class_='description')
print (description[0].text)
+71
View File
@@ -0,0 +1,71 @@
import psycopg2
import wget
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
import time
i = 1
s = Service("С:\DATA\ChromeDriver\chromedriver.exe")
browser = webdriver.Chrome(service=s)
browser.get("https://www.mvideo.ru/noutbuki-planshety-komputery-8/noutbuki-118?from=under_search")
height = browser.execute_script("return document.body.scrollHeight")
time.sleep(4)
while 900*i < height:
height = browser.execute_script("return document.body.scrollHeight")
browser.execute_script(f"window.scrollTo(0,{900*i})")
time.sleep(1)
i+=1
html_text = browser.page_source
soup = BeautifulSoup(html_text, 'lxml')
productst = soup.find_all('a', class_="product-title__text")
pricest = soup.find_all('span', class_="price__main-value")
features = soup.find_all('ul', class_="product-feature-list product-feature-list--undefined")
pictures = soup.find_all('img', class_="product-picture__img product-picture__img--list")
products, prices, diagonals, resolutions, CPUs, RAMs, Graphics_controllers, Volumes = [],[],[],[],[],[],[],[]
for i in range(len(productst)):
products.append(productst[i].text.strip())
prices.append(pricest[i].text.replace("\xa0", "").strip())
for feature in features:
u = feature.text
diagonals.append(u[20:u.find("\"",20)+1])
resolutions.append(u[u.find("\"",20)+2:u.find(".", 23)+1])
CPUs.append(u[u.find("Процессор",20)+9:u.find("ГГц",20)+3].replace("\xa0",""))
RAMs.append(" ".join(u[u.find("(RAM)",20)+5:u.find("Графический контроллер")].split()))
Graphics_controllers.append(u[u.find("Графический контроллер")+22:u.rfind("Объем")-2])
Volumes.append(u[u.rfind("Объем")+6:].replace("\xa0", "").strip())
for i in range(len(Volumes)):
u = Volumes[i]
if u.rfind("D")!=-1: Volumes[i] = u[:u.rfind("D")+1]+ " " + u[u.rfind("D")+1:]
else: Volumes[i] = u[:u.rfind("С")+1]+ " " + u[u.rfind("С")+1:]
connection=psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
cursor = connection.cursor()
insert = """CREATE TABLE public.laptops(
id serial primary key,
product varchar(100),
price varchar(15),
diagonal varchar(5),
resolution varchar(20),
CPU varchar(50),
RAM varchar(10),
Graphics_Controller varchar(40),
Volume varchar(25),
src varchar(100)
);
"""
cursor.execute(insert)
connection.commit()
for i in range(len(products)):
url = "https://"+pictures[i]['src']
filename = f"C:\DATA\Images\{i+1}.jpg"
wget.download(url, filename)
insert = f"""INSERT INTO public.laptops(
product, price, diagonal, resolution, CPU, RAM, Graphics_Controller, Volume, src)
VALUES
('{products[i]}', '{prices[i]}', '{diagonals[i]}', '{resolutions[i]}', '{CPUs[i]}', '{RAMs[i]}', '{Graphics_controllers[i]}', '{Volumes[i]}', '{filename}');"""
cursor.execute(insert)
connection.commit()
cursor.execute("select * from laptops")
print("Результат", cursor.fetchall())
cursor.close()
connection.close()
@@ -0,0 +1,42 @@
import psycopg2
import wget
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
s = Service("С:\DATA\ChromeDriver\chromedriver.exe")
browser = webdriver.Chrome(service=s)
browser.get("https://online.metro-cc.ru/virtual/assortiment_rioba-4887?from=under_search&page=2")
html_text = browser.page_source
soup = BeautifulSoup(html_text, 'lxml')
productst = soup.find_all('span', class_="product-card-name__text")
pricest = soup.find_all('span', class_="product-price__sum-rubles")
pictures = soup.find_all('img', class_="product-card-photo__image")
connection=psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
cursor = connection.cursor()
products, prices = [], []
for i in range(len(productst)):
products.append(productst[i].text.replace("\n", "").strip())
prices.append(pricest[i].text+" Р/шт")
insert = """CREATE TABLE public.Chocolate(
id serial primary key,
product varchar(100),
price varchar(25),
src varchar(50)
);
"""
cursor.execute(insert)
connection.commit()
for i in range(len(products)):
url = pictures[i]['src']
filename = f"C:\DATA\Images2\{i+1}.jpg"
wget.download(url, filename)
insert = f"""INSERT INTO public.Chocolate(
product, price, src)
VALUES
('{products[i]}', '{prices[i]}', '{filename}');"""
cursor.execute(insert)
connection.commit()
cursor.execute("select * from Chocolate")
print("Результат", cursor.fetchall())
cursor.close()
connection.close()
File diff suppressed because one or more lines are too long
+1
View File
@@ -0,0 +1 @@
test
@@ -0,0 +1,41 @@
import psycopg2
import wget
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
cursor = connection.cursor()
cr_qwery = """ create table Parser
(id serial primary key, name varchar(100), sale varchar(30), price varchar(10), rate varchar(10), picture varchar(100))"""
#cursor.execute(cr_qwery)
connection.commit()
driver = Service('D:\GOGdriver\chromedriver.exe')
browser = webdriver.Chrome(service=driver)
browser.get('https://www.perekrestok.ru/cat/promo/10')
html_code = browser.page_source
soup = BeautifulSoup(html_code, 'lxml')
name = soup.find_all('div', class_="product-card__title")
sale = soup.find_all('div', class_="sc-kYrlTI ciSrDA product-card__badge")
price = soup.find_all('div', class_="price-new")
rate = soup.find_all('div', class_="rating-value")
picture = soup.find_all('div', class_="product-card__image-wrapper")
for i in range(15):
url = picture[i].find('img').attrs['src']
filename = f"C:\\Users\\71332\PycharmProjects\pythonProject1\\venv\Pictures\{i}.jpg"
wget.download(url, filename)
ins_qwery = f"""insert into public.Parser(name, sale, price, rate, picture) values ('{name[i].text}', '{sale[i].text}', '{price[i].text}', '{rate[i].text}', '{filename}')"""
cursor.execute(ins_qwery)
connection.commit()
cursor.close()
connection.close()
+16
View File
@@ -0,0 +1,16 @@
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
driver = Service('D:\GOGdriver\chromedriver.exe')
browser = webdriver.Chrome(service=driver)
browser.get('https://www.kinopoisk.ru/lists/movies/popular-films/')
html_code = browser.page_source
soup = BeautifulSoup(html_code, 'lxml')
name = soup.find_all('div', class_="desktop-list-main-info_secondaryTitleSlot__mc0mI")
print(soup)
print(name)
for i in name:
print(i.text)
+14
View File
@@ -0,0 +1,14 @@
from bs4 import BeautifulSoup
from selenium.webdriver import Chrome
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
import time
s = Service('C:\\Users\\Yekaterina\\Downloads\\chromedriver_win32\\chromedriver.exe')
browser = webdriver.Chrome(service=s)
browser.get('https://www.citilink.ru/catalog/noutbuki/')
html_text = browser.page_source
soup = BeautifulSoup(html_text, 'lxml')
name=soup.find_all('div', class_='app-catalog-1tp0ino e1an64qs0')
print (name[0].text)
description=soup.find_all('div', class_='app-catalog-1o4umte eevw8x70')
print (description[0].text)
+17 -29
View File
@@ -1,40 +1,28 @@
# установи python, pip
# выполни эту команду: pip install selenium bs4 lxml
# если ты используешь pycharm, то надо библиотеки ставить не в консоли(терминале), а в самом pycharm
# для этого нажми "view" > "tool windows" > "python packages"
# внизу в окне поиска ищи нужные библиотеки
# когда найдешь, нажимай на нее, справа будет кнопка "install package", устанавливай, потом перезапусти pycharm
# найди версию своего браузера
# напиши в гугле "скачать драйвер для <название и версия твоего браузера> selenium"
# теперь все готово для запуска этого
# помните, что сам сайт(который вы хотите парсить) может вас блокировать или заставлять проходить capture, из-за чего этот код ничего не выведет
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
# импортируем всё нужное
from wget import download
import psycopg2 as psyc
s = Service("путь_до_драйвера") # в ковычках указываем полный путь до скаченного ранее драйвера
# если ты на винде, то вместо знака "\" пиши "\\"
brow = webdriver.Chrome(service=s) # как будто создаем виртуальный браузер
brow = webdriver.Chrome()
url = "https://ikey.ru/"
brow.get(url)
brow.get("https://www.revshells.com/") # получаем html код сайта и другую информацию(она нам не нужна сейчас)
html = brow.page_source
html = brow.page_source # копируем html код в переменную
soup = BeautifulSoup(html, "lxml")
soup = BeautifulSoup(html, "lxml") # создаем специальный парсер
products = soup.find_all(attrs={"class": "product"})
buttons = soup.find_all(attrs={"class": "list-group-item list-group-item-action"})
# получаем список всех html тегов, в которых есть атрибут "class", равный: "list-group-item list-group-item-action"
with psyc.connect(dbname="db_for_parse", user="perfecto") as conn:
with conn.cursor() as cursor:
for i, product in enumerate(products):
img_tag = product.find(attrs={"class": "imagef"}).find("img")
title = img_tag.attrs.get("alt")
image = url + img_tag.attrs.get("src")
download(url + img_tag.attrs.get("src"), f"images/{i}.jpg")
for button in buttons:
# выводим текст каждого тега
print(button.text)
# от сердца и почек
# дарю вам питончик
# made by perfecto
cursor.execute(f"insert into images(link, name) values ('{image}', '{title}')")
conn.commit()
+1 -1
View File
@@ -1 +1 @@
def print_hi(' hello')
абоба
@@ -0,0 +1,14 @@
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
s = Service('C:\Desktop\exe\chromedriver.exe')
browser = webdriver.Chrome(service=s)
browser.get('https://www.hellride.ru/catalog/zapchasti-dlya-tryukovyh-samokatov/deki')
html_text = browser.page_source
soup = BeautifulSoup(html_text, 'lxml')
prices = soup.find_all(attrs={"class": "product-card__price"})
titles = soup.find_all(attrs={"class": "product-card__title"})
for price, title in zip(prices, titles):
print(f" название {title.text} ; цена: {price.text}")