Merge branch 'master' of github.com:KUlishevgeniy/c22712

.idea/misc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.11" project-jdk-type="Python SDK" />
 </project>

.idea/workspace.xml

@@ -4,9 +4,9 @@
     <option name="autoReloadType" value="SELECTIVE" />
   </component>
   <component name="ChangeListManager">
-    <list default="true" id="e5615680-c090-497d-a104-1337ed85f850" name="Changes" comment="Первый коммит">
+    <list default="true" id="e5615680-c090-497d-a104-1337ed85f850" name="Changes" comment="ганьг или баньг&#10;СХС">
-      <change afterPath="$PROJECT_DIR$/Задания/task1/Lahin/parsing.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/main.py" beforeDir="false" afterPath="$PROJECT_DIR$/main.py" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -38,13 +38,17 @@
   </component>
   <component name="PropertiesComponent">{
   &quot;keyToString&quot;: {
-    &quot;RunOnceActivity.OpenProjectViewOnStart&quot;: &quot;true&quot;,
+    &quot;last_opened_file_path&quot;: &quot;C:/Users/rusta/c22712&quot;,
-    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
+    &quot;settings.editor.selected.configurable&quot;: &quot;com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable&quot;
-    &quot;WebServerToolWindowFactoryState&quot;: &quot;false&quot;
   }
 }</component>
-  <component name="RunManager">
+  <component name="RecentsManager">
-    <configuration name="main" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="C:\Users\rusta\c22712" />
+    </key>
+  </component>
+  <component name="RunManager" selected="Python.main">
+    <configuration name="1page1win" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
       <module name="22712" />
       <option name="INTERPRETER_OPTIONS" value="" />
       <option name="PARENT_ENVS" value="true" />
@@ -52,8 +56,30 @@
         <env name="PYTHONUNBUFFERED" value="1" />
       </envs>
       <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/../OneDrive/Рабочий стол/парсинг" />
+      <option name="IS_MODULE_SDK" value="false" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/../OneDrive/Рабочий стол/парсинг/1page1win.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <configuration name="main" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
+      <module name="22712" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="C:\Users\rusta\PycharmProjects\pythonProject\venv\Scripts\python.exe" />
+      <option name="SDK_NAME" value="Python 3.11 (pythonProject)" />
       <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
-      <option name="IS_MODULE_SDK" value="true" />
+      <option name="IS_MODULE_SDK" value="false" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
       <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
@@ -66,6 +92,11 @@
       <option name="INPUT_FILE" value="" />
       <method v="2" />
     </configuration>
+    <recent_temporary>
+      <list>
+        <item itemvalue="Python.1page1win" />
+      </list>
+    </recent_temporary>
   </component>
   <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
   <component name="TaskManager">
@@ -105,7 +136,21 @@
       <option name="project" value="LOCAL" />
       <updated>1676532798232</updated>
     </task>
-    <option name="localTasksCounter" value="5" />
+    <task id="LOCAL-00005" summary="parser">
+      <created>1676934066947</created>
+      <option name="number" value="00005" />
+      <option name="presentableId" value="LOCAL-00005" />
+      <option name="project" value="LOCAL" />
+      <updated>1676934066947</updated>
+    </task>
+    <task id="LOCAL-00006" summary="ганьг или баньг&#10;СХС">
+      <created>1678906921509</created>
+      <option name="number" value="00006" />
+      <option name="presentableId" value="LOCAL-00006" />
+      <option name="project" value="LOCAL" />
+      <updated>1678906921509</updated>
+    </task>
+    <option name="localTasksCounter" value="7" />
     <servers />
   </component>
   <component name="TypeScriptGeneratedFilesManager">
@@ -124,14 +169,15 @@
   </component>
   <component name="VcsManagerConfiguration">
     <MESSAGE value="Первый коммит" />
-    <option name="LAST_COMMIT_MESSAGE" value="Первый коммит" />
+    <MESSAGE value="parser" />
+    <MESSAGE value="ганьг или баньг&#10;СХС" />
+    <option name="LAST_COMMIT_MESSAGE" value="ганьг или баньг&#10;СХС" />
   </component>
   <component name="XDebuggerManager">
     <breakpoint-manager>
       <breakpoints>
         <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
           <url>file://$PROJECT_DIR$/main.py</url>
-          <line>8</line>
           <option name="timeStamp" value="1" />
         </line-breakpoint>
       </breakpoints>

main.py

@@ -1,16 +1,23 @@
-# This is a sample Python script.
+from bs4 import BeautifulSoup
+from selenium.webdriver import Chrome
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+import time
 
-# Press Shift+F10 to execute it or replace it with your code.
+# Selenium - библиотека для автоматизации действий веб браузера, скрапинга
-# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
+# запускаем браузер
+s = Service('C:\data\chrome\chromedriver.exe')
+browser = webdriver.Chrome(service=s)
+browser.get('https://www.wildberries.ru/catalog/muzhchinam/odezhda/futbolki-i-mayki?bid=3b73ce5a-8eba-45a1-b9b2-0723f7592eac#c145658286')
+time.sleep (10)#задержка для ввода капчи
+html_text = browser.page_source
+soup = BeautifulSoup(html_text, 'lxml')
+films=soup.find_all('div', class_='/games/the-elder-scrolls-v-skyrim/')
+ratings = soup.find_all('div', class_='KnbCardMark_label__hg6Pg tv-series-mark KnbCardMark_isGreen__G_FLl')
+for film, rating in zip(films, ratings):
+    print(f"Игра:{film.text} | Рейтинг:{rating.text}")
 
 
-def print_hi(name):
+# Это пример парсинга. Вам необходимо спарсить 1 страницу каталога любого сайта на выбор.
-    # Use a breakpoint in the code line below to debug your script.
+#сайты не должны повторяться
-    print(f'Hi, {name}')  # Press Ctrl+F8 to toggle the breakpoint.
+# Спарсить необходимо только Заголовки и описание.
-
-
-# Press the green button in the gutter to run the script.
-if __name__ == '__main__':
-    print_hi('PyCharm')
-
-# See PyCharm help at https://www.jetbrains.com/help/pycharm/

task1/Задание.py

@@ -0,0 +1,23 @@
+
+
+from bs4 import BeautifulSoup
+from selenium.webdriver import Chrome
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+import time
+
+# Selenium - библиотека для автоматизации действий веб браузера, скрапинга
+# запускаем браузер
+s = Service('C:\data\hrome\chromedriver.exe')
+browser = webdriver.Chrome(service=s)
+browser.get('https://www.kinopoisk.ru/lists/movies/top250/')
+time.sleep (10)#задержка для ввода капчи
+html_text = browser.page_source
+soup = BeautifulSoup(html_text, 'lxml')
+films=soup.find_all('div', class_='base-movie-main-info_mainInfo__ZL_u3')
+print (films[0].text)
+
+
+# Это пример парсинга. Вам необходимо спарсить 1 страницу каталога любого сайта на выбор.
+#сайты не должны повторяться
+# Спарсить необходимо только Заголовки и описание.

venv/task 1/task1.py

@@ -1 +0,0 @@
-Тестовый  вап

Задания/task1/Antipenko/DB1.py

@@ -0,0 +1,42 @@
+import psycopg2
+import wget
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+# Парсер и загрузчик в БД//Антипенко Дмитрий
+connection = psycopg2.connect(host='localhost', dbname='PythonDB', user='postgres', password='Q1w2e3r4')
+
+cursor = connection.cursor()
+
+creat_qwery = """ create table Parser 
+    (id serial primary key, page_name varchar(100), price varchar(10), priceDis varchar(30), mark varchar(10), scr varchar(100))"""
+
+cursor.execute(creat_qwery)
+connection.commit()
+
+driver = Service('D:\teach\Prog\chromedriver.exe') 
+browser = webdriver.Chrome(service=driver)
+browser.get('https://amwine.ru/catalog/igristoe_vino_i_shampanskoe/igristoe_vino/')
+html_code = browser.page_source
+b_soup = BeautifulSoup(html_code, 'lxml')
+name = b_soup.find_all('a', class_="catalog-list-item__title js-product-detail-link")
+price = b_soup.find_all('span', class_="middle_price")
+priceDis = b_soup.find_all('span', class_="baseoldprice")
+mark = b_soup.find_all('span', class_="product-rating__rating")
+pictures = b_soup.find_all('div', class_="catalog-list-item__img-wrapper")
+
+for i in range(15):
+    url = 'https://amwine.ru'+pictures[i].find('a').find('img').attrs['data-src']
+    filename = f"Programming\\23.03\img\{i}.jpg"
+    print(filename)
+    wget.download(url, filename)
+    ins_qwery = f"""insert into public.Parser(page_name, price, priceDis, mark, scr) values ('{name[i].text}', '{price[i].text}', '{priceDis[i].text}',  '{mark[i].text}', '{filename}')"""
+    cursor.execute(ins_qwery)
+    connection.commit()
+
+
+
+
+cursor.close()
+
+connection.close()

Задания/task1/Antipenko/ada.main.py

@@ -1,3 +1,6 @@
+
+#Вариант с пары
+'''
 from bs4 import BeautifulSoup
 from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
@@ -10,9 +13,27 @@ html_text = browser.page_source
 time.sleep(20)
 soup = BeautifulSoup(html_text, 'lxml')
 films = soup.find_all('div', class_='base-movie-main-info_mainInfo__ZL_u3')
-'''
+
 print(soup)
 print(films)
-'''
+
 for film in films:
     print(film.text)
+'''
+#Домашка
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+
+driver = Service('D:\teach\Prog\chromedriver.exe') 
+browser = webdriver.Chrome(service=driver)
+browser.get('https://hmbrussia.ru/regional-office/')
+html_code = browser.page_source
+b_soup = BeautifulSoup(html_code, 'lxml')
+name = b_soup.find_all('div', class_="ps-xl-3 ms-xl-3")
+
+print(b_soup)
+print(name)
+
+for i in name:
+    print(i.text)

Задания/task1/Epifanov/1 - st hw.py

@@ -0,0 +1,14 @@
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+
+s = Service('C:\Desktop\exe\chromedriver.exe')
+browser = webdriver.Chrome(service=s)
+browser.get('https://www.rftoday.ru/_vse_zagolovki2')
+html_text = browser.page_source
+soup = BeautifulSoup(html_text, 'lxml')
+infos = soup.find_all(attrs={"class": "title"})
+revs = soup.find_all(attrs={"class": "source"})
+#print(infos[0].text) - если нужно вывести только n-ый заголовок
+for info, rev in zip(infos, revs):
+    print(f"{info.text} ; Источник: {rev.text}")#Вывод всех заголовков новостей и их источников со страницы 1

Задания/task1/Filcheva/Verst1.py

@@ -0,0 +1,12 @@
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from time import sleep
+s = Service("C:\Driver\chromedriver.exe")
+browser = webdriver.Chrome(service=s)
+browser.get("https://mephi.ru/")
+sleep(10)
+html_text = browser.page_source
+soup = BeautifulSoup(html_text, "lxml")
+serv = soup.find_all("div", class_="views-field views-field-title menu-item menu-item-10702")
+print(serv[0].text)

Задания/task1/Garanyan/main.py

@@ -0,0 +1,12 @@
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+
+s = Service("E:\data\chromedriver.exe")
+browser = webdriver.Chrome(service=s)
+browser.get("https://ria.ru/")
+html_text = browser.page_source
+soup = BeautifulSoup(html_text, 'lxml')
+news = soup.find_all('div', class_='cell cell-list')
+for new in news:
+    print(new.text)

Задания/task1/Gasanova/parser.py

@@ -0,0 +1,9 @@
+#made by dialuna (or just Diana Gasanova) <3
+import requests
+from bs4 import BeautifulSoup as bs
+URL = "https://www.kinopoisk.ru/lists/movies/top-250-2020/"
+r = requests.get(URL, timeout=10)
+soup = bs(r.text, "html.parser") 
+films = soup.find_all(attrs={"class": "styles_mainTitle__IFQyZ styles_activeMovieTittle__kJdJj"})
+for film in films:
+    print(film.text)

Задания/task1/Kluchinskaya1/File

@@ -1 +0,0 @@
-text

Задания/task1/Kluchinskaya1/New commit for parsing

@@ -0,0 +1,12 @@
+ from bs4 import BeautifulSoup
+ from selenium import webdriver
+ from selenium.webdriver.chrome.service import Service
+ s = Service('C:\\Users\\Dasha\\Desktop\\Progect\\chromedriver.exe')
+browser = webdriver.Chrome(service=s)
+ browser.get('https://www.livelib.ru/selection/907286-vybor-redaktsii-elle-girl')
+ html_text=browser.page_source
+ soup=BeautifulSoup(html_text, 'lxml')
+ infos = soup.find_
+ films=soup.find_all(attrs={"class": "brow-book-name with-cycle"})
+ for film in films:
+	print(film.text)

Задания/task1/Kulikov/main.py

@@ -0,0 +1,25 @@
+# pip install selenium bs4
+
+
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from time import sleep
+
+
+s = Service("C:\\Users\\Alex\\PycharmProjects\\pythonProject\\chromedriver.exe")
+brow = webdriver.Chrome(service=s)
+
+# brow.get("https://www.kinopoisk.ru/lists/movies/top250/")
+brow.get("https://www.revshells.com/")
+sleep(10)
+
+html = brow.page_source
+
+soup = BeautifulSoup(html, "lxml")
+
+# films = soup.find_all(attrs={"class": "styles_mainTitle__IFQyZ styles_activeMovieTittle__kJdJj"})
+buttons = soup.find_all(attrs={"class": "list-group-item list-group-item-action"})
+
+for button in buttons:
+    print(button.text)

Задания/task1/Kuznetsov/parcing.py

@@ -0,0 +1,15 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver import Chrome
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+import time
+s = Service('E:\\Downloads\\chromedriver_win32.zip\\chromedriver.exe')
+browser = webdriver.Chrome(service=s)
+browser.get('https://trial-sport.ru/gds.php?s=51516&c1=1070639&c2=1070640')
+html_text = browser.page_source
+soup = BeautifulSoup(html_text, 'lxml')
+product_name=soup.find_all('a', class_='title')
+description=soup.find_all('span', class_='description')
+for product_name, description in zip(product_name, description):
+    print(f"Продукт: {product_name.text} \nОписание: {description.text}")
+print(" ")

Задания/task1/Kuznetsov/test

@@ -1 +1,14 @@
-test
+from bs4 import BeautifulSoup
+from selenium.webdriver import Chrome
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+import time
+s = Service('E:\\Downloads\\chromedriver_win32.zip\\chromedriver.exe')
+browser = webdriver.Chrome(service=s)
+browser.get('https://trial-sport.ru/gds.php?s=51516&c1=1070639&c2=1070640')
+html_text = browser.page_source
+soup = BeautifulSoup(html_text, 'lxml')
+bike_name=soup.find_all('a', class_='title')
+print (bike_name[0].text)
+description=soup.find_all('span', class_='description')
+print (description[0].text)

Задания/task1/Lahin/parsing.py

@@ -1,14 +1,14 @@
+
 from bs4 import BeautifulSoup
 from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
-from time import sleep
 s = Service("С:\DATA\ChromeDriver\chromedriver.exe")
 browser = webdriver.Chrome(service=s)
-browser.get("https://www.kinopoisk.ru/lists/movies/top250/")
+browser.get("https://online.metro-cc.ru/category/sladosti-chipsy-sneki/shokolad-batonchiki?from=under_search&is_action=1")
-sleep(15)
 html_text = browser.page_source
 soup = BeautifulSoup(html_text, 'lxml')
-films = soup.find_all('div', class_="desktop-list-main-info_secondaryTitleSlot__mc0mI")
+products = soup.find_all('a', class_="product-card-name reset-link catalog-2-level-product-card__name style--catalog-2-level-product-card")
-for film in films:
+prices = soup.find_all('span', class_="product-card-prices__actual color-red")
-    print(film.text)
+for product,price in zip(products, prices):
-    print("--------")
+    print(f"Продукт: {product.text[3:]}Цена: {price.text}")
+    print("-"*100)

Задания/task1/Samigullin/test

@@ -0,0 +1 @@
+test

Задания/task1/Uskova/parcing.py

@@ -0,0 +1,14 @@
+from bs4 import BeautifulSoup
+from selenium.webdriver import Chrome
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+import time
+s = Service('C:\\Users\\Yekaterina\\Downloads\\chromedriver_win32\\chromedriver.exe')
+browser = webdriver.Chrome(service=s)
+browser.get('https://www.citilink.ru/catalog/noutbuki/')
+html_text = browser.page_source
+soup = BeautifulSoup(html_text, 'lxml')
+name=soup.find_all('div', class_='app-catalog-1tp0ino e1an64qs0')
+print (name[0].text)
+description=soup.find_all('div', class_='app-catalog-1o4umte eevw8x70')
+print (description[0].text)

Задания/task1/nikitin/test2.py

@@ -0,0 +1,12 @@
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+s = Service('D:\Games\data\chromedriver.exe')
+browser = webdriver.Chrome(service=s)
+browser.get('https://www.volkswagen.ru/polo/')
+html_text = browser.page_source
+soup = BeautifulSoup(html_text, 'lxml')
+car_names = soup.find_all('div', class_='avn001-2_name')
+prices = soup.find_all('div', class_='avn001-2_price-container')
+for car_name, price in zip(car_names, prices):
+    print(f"Название машины:{car_name.text} | Цена: {price.text} рублей")