for parsing in db

This commit is contained in:
Sklvd
2023-03-25 15:12:50 +03:00
parent df2c5e857c
commit 2cf82629d9
2 changed files with 54 additions and 4 deletions
+28 -4
View File
@@ -4,7 +4,10 @@
<option name="autoReloadType" value="SELECTIVE" />
</component>
<component name="ChangeListManager">
<list default="true" id="e5615680-c090-497d-a104-1337ed85f850" name="Changes" comment="parser" />
<list default="true" id="e5615680-c090-497d-a104-1337ed85f850" name="Changes" comment="for parsing">
<change afterPath="$PROJECT_DIR$/Задания/task1/Kluchinskaya1/parsing_in_bd.py" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
@@ -22,7 +25,6 @@
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
<option name="UPDATE_TYPE" value="REBASE" />
</component>
<component name="MarkdownSettingsMigration">
<option name="stateVersion" value="1" />
@@ -33,6 +35,11 @@
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent">
<property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
<property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component>
<component name="RunManager">
<configuration name="main" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
<module name="22712" />
@@ -102,7 +109,21 @@
<option name="project" value="LOCAL" />
<updated>1676934066947</updated>
</task>
<option name="localTasksCounter" value="6" />
<task id="LOCAL-00006" summary="for parsing">
<created>1677183215124</created>
<option name="number" value="00006" />
<option name="presentableId" value="LOCAL-00006" />
<option name="project" value="LOCAL" />
<updated>1677183215124</updated>
</task>
<task id="LOCAL-00007" summary="for parsing">
<created>1677183242833</created>
<option name="number" value="00007" />
<option name="presentableId" value="LOCAL-00007" />
<option name="project" value="LOCAL" />
<updated>1677183242833</updated>
</task>
<option name="localTasksCounter" value="8" />
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
@@ -118,11 +139,14 @@
</entry>
</map>
</option>
<option name="oldMeFiltersMigrated" value="true" />
</component>
<component name="VcsManagerConfiguration">
<MESSAGE value="Первый коммит" />
<MESSAGE value="parser" />
<option name="LAST_COMMIT_MESSAGE" value="parser" />
<MESSAGE value=" for parsing" />
<MESSAGE value="for parsing" />
<option name="LAST_COMMIT_MESSAGE" value="for parsing" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
@@ -0,0 +1,26 @@
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from wget import download
import psycopg2
connection = psycopg2.connect(host='localhost', dbname='dbdata',
user='postgres', password='Q1w2e3r4')
cursor=connection.cursor()
s = Service('C:\\Users\\Dasha\\Desktop\\Progect\\chromedriver.exe')
browser = webdriver.Chrome(service=s)
url = 'https://www.tretyakovgallery.ru/'
browser.get(url)
html_text=browser.page_source
soup = BeautifulSoup(html_text, 'lxml')
products = soup.find_all(attrs={"class": "building-item"})
for i, product in enumerate(products):
link = product.find("img").attrs.get("src")
tag = product.find("strong")
if tag is not None:
download(url + link, f"img\\{i}.jpg")
cursor.execute(f"""INSERT INTO public.parsing(name, link)
VALUES
( '{tag.text}', '{url + link}')""")
connection.commit()
cursor.close()
connection.close()