Merge remote-tracking branch 'origin/master'

Задания/task1/Epifanov/FHWDB.py

@@ -5,28 +5,30 @@ import psycopg2
 import wget
 
 browser = webdriver.Chrome(service=Service('C:\Desktop\exe\chromedriver.exe'))
-browser.get('https://flawery.ru/moscow/bouquets/event-yanvary25/?min=0&max=max&sorting=discount&filter_24h=on')
+browser.get('https://flawery.ru/moscow/bouquets/color-blue/?min=0&max=max&sorting=discount')
 soup = BeautifulSoup(browser.page_source, "lxml")
 Name = soup.find_all(attrs={"class": "catalog_title"})
 Price = soup.find_all(attrs={"class": "catalog_price_now"})
-Delivery_Time = soup.find_all(attrs={"class": "catalog_express"})
 Discount = soup.find_all(attrs={"class": "catalog_sale"})
 Image = soup.find_all(attrs={"class": "catalog_item catalog_item_popup"})
 
 connection = psycopg2.connect(host='localhost', dbname='FHWDB', user='postgres', password='Q1w2e3r4')
 cursor = connection.cursor()
 create_q = '''CREATE TABLE Parse  
-     (ID serial primary key, Name varchar(100), Price varchar(9), Delivery_Time varchar(25), Discount varchar(10), src varchar(110))'''
+     (ID serial primary key, Name varchar(100), Price varchar(9), Discount varchar(10), src_of_picture varchar(50))'''
 cursor.execute(create_q)
 connection.commit()
 
 for j in range(60):
     url = 'https://flawery.ru'+Image[j].find('a').find('picture').find('img').attrs['src']
-    tempf = f"C:\\Users\\user\\Desktop\\FHWDB{j}.jpg"
+    tempf = f"C:\\Users\\user\\PFDB{j}.jpg"
     wget.download(url, tempf)
-    insert_query = f'''INSERT into public.Parse(Name, Price, Delivery_Time, Discount, src) values ('{Name[j].text}', '{Price[j].text}', '{Delivery_Time[j].text}', '{Discount[j].text}', '{tempf}') '''
+    insert_query = f'''INSERT into public.Parse(Name, Price, Discount, src_of_picture) values ('{Name[j].text}', '{Price[j].text}', '{Discount[j].text}', '{tempf}') '''
     cursor.execute(insert_query)
-
 connection.commit()
+
+cursor.execute("SELECT * from Parse")
+print(cursor.fetchall())
+
 cursor.close()
 connection.close()

Задания/task1/Garanyan/main.py

@@ -1,12 +1,35 @@
 from bs4 import BeautifulSoup
 from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
+import psycopg2
+import wget
 
-s = Service("E:\data\chromedriver.exe")
+browser = webdriver.Chrome(service=Service('E:\data\chromedriver.exe'))
-browser = webdriver.Chrome(service=s)
+browser.get('https://8956food.ru/')
-browser.get("https://ria.ru/")
+soup = BeautifulSoup(browser.page_source, "lxml")
-html_text = browser.page_source
+
-soup = BeautifulSoup(html_text, 'lxml')
+Name = soup.find_all(attrs={"class": "subtitle"})
-news = soup.find_all('div', class_='cell cell-list')
+Price = soup.find_all(attrs={"class": "price"})
-for new in news:
+Image = soup.find_all(attrs={"class": "holder-img"})
-    print(new.text)
+
+connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
+cursor = connection.cursor()
+create_q = '''CREATE TABLE Food  
+     (ID serial primary key, Name varchar(500), Price varchar(60), src varchar(100))'''
+cursor.execute(create_q)
+connection.commit()
+
+#https://smartomato.ams3.cdn.digitaloceanspaces.com/uploads/media/photo/769221/dish_large__1.jpg
+for j in range(15):
+    url = Image[j].find('img').attrs['src']
+    tempf = f"C:\\Users\\Harutyun\\Desktop\\prog\\food{j}.jpg"
+    wget.download(url, tempf)
+    insert_query = f'''INSERT into public.Food(Name, Price, src) values ('{Name[j].text}', '{Price[j].text}', '{tempf}') '''
+    cursor.execute(insert_query)
+connection.commit()
+
+cursor.execute("SELECT * from Food")
+print(cursor.fetchall())
+
+cursor.close()
+connection.close()

Задания/task1/Kluchinskaya1/parsing_2.py

@@ -0,0 +1,29 @@
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from wget import download
+import psycopg2
+connection = psycopg2.connect(host='localhost', dbname='dbdata',
+                              user='postgres', password='Q1w2e3r4')
+cursor=connection.cursor()
+s = Service('C:\\Users\\Dasha\\Desktop\\Progect\\chromedriver.exe')
+browser = webdriver.Chrome(service=s)
+browser = webdriver.Chrome()
+url = 'https://www.tretyakovgallery.ru/'
+browser.get(url)
+html_text=browser.page_source
+soup = BeautifulSoup(html_text, 'lxml')
+products = soup.find_all(attrs={"class": "building-item"})
+for i, product in enumerate(products):
+    link = product.find("img").attrs.get("src")
+    tag = product.find("strong")
+    name = product.find(attrs={"class": "building-item__title"})
+    description = product.find(attrs={"class": "building-item__description"})
+    if tag is not None:
+        download(url + link, f"img\\{i}.jpg")
+        cursor.execute(f"""INSERT INTO public.parsing(url, workhours, name, description, filename)
+                          VALUES           
+                          ( '{url + link}', '{tag.text}', '{name.text.strip()}', '{description.text.strip()}', 'img\\{i}.jpg')""")
+connection.commit()
+cursor.close()
+connection.close()

Задания/task1/Lahin/postgres.py

@@ -7,65 +7,55 @@ import time
 i = 1
 s = Service("С:\DATA\ChromeDriver\chromedriver.exe")
 browser = webdriver.Chrome(service=s)
-browser.get("https://www.mvideo.ru/noutbuki-planshety-komputery-8/noutbuki-118?from=under_search")
+browser.get("https://www.mvideo.ru/noutbuki-planshety-komputery-8/noutbuki-118/f/tolko-v-nalichii=da?from=under_search")
+time.sleep(5)
+"""
 height = browser.execute_script("return document.body.scrollHeight")
 time.sleep(4)
-while 900*i < height:
+for i in range(10):
-    height = browser.execute_script("return document.body.scrollHeight")
+    browser.execute_script(f"window.scrollTo(0, {1000*i})")
-    browser.execute_script(f"window.scrollTo(0,{900*i})")
     time.sleep(1)
-    i+=1
+"""
 html_text = browser.page_source
 soup = BeautifulSoup(html_text, 'lxml')
 productst = soup.find_all('a', class_="product-title__text")
 pricest = soup.find_all('span', class_="price__main-value")
-features = soup.find_all('ul', class_="product-feature-list product-feature-list--undefined")
+features = soup.find_all('span', class_="product-feature-list__value")
-pictures =  soup.find_all('img', class_="product-picture__img product-picture__img--list")
+pictures =  soup.find_all('div', class_="product-picture-container")
-products, prices, diagonals, resolutions, CPUs, RAMs, Graphics_controllers, Volumes = [],[],[],[],[],[],[],[]
-for i in range(len(productst)):
-    products.append(productst[i].text.strip())
-    prices.append(pricest[i].text.replace("\xa0", "").strip())
-for feature in features:
-    u = feature.text
-    diagonals.append(u[20:u.find("\"",20)+1])
-    resolutions.append(u[u.find("\"",20)+2:u.find(".", 23)+1])
-    CPUs.append(u[u.find("Процессор",20)+9:u.find("ГГц",20)+3].replace("\xa0",""))
-    RAMs.append(" ".join(u[u.find("(RAM)",20)+5:u.find("Графический контроллер")].split()))
-    Graphics_controllers.append(u[u.find("Графический контроллер")+22:u.rfind("Объем")-2])
-    Volumes.append(u[u.rfind("Объем")+6:].replace("\xa0", "").strip())
-for i in range(len(Volumes)):
-    u = Volumes[i]
-    if u.rfind("D")!=-1: Volumes[i] = u[:u.rfind("D")+1]+ " " + u[u.rfind("D")+1:]
-    else: Volumes[i] = u[:u.rfind("С")+1]+ " " + u[u.rfind("С")+1:]
 connection=psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
 cursor = connection.cursor()
-insert = """CREATE TABLE public.laptops(
+""" insert = CREATE TABLE public.laptops(
 id serial primary key,
 product varchar(100),
 price varchar(15),
 diagonal varchar(5),
 resolution varchar(20),
 CPU varchar(50),
-RAM varchar(10),
+RAM varchar(15),
 Graphics_Controller varchar(40),
 Volume varchar(25),
 src varchar(100)
 );
 """
+insert = """TRUNCATE TABLE public.laptops; ALTER SEQUENCE laptops_id_seq RESTART WITH 1;"""
 cursor.execute(insert)
 connection.commit()
-for i in range(len(products)):
+for i in range(len(productst)):
-    url = "https://"+pictures[i]['src']
+    url = "https://"+pictures[i].find('img')['src']
     filename = f"C:\DATA\Images\{i+1}.jpg"
+    u = []
+    for j in range(10):
+        u.append(features[10*i+j].text)
     wget.download(url, filename)
+    t = pricest[i].text.replace("\xa0", " ")
     insert = f"""INSERT INTO public.laptops(
         product, price, diagonal, resolution, CPU, RAM, Graphics_Controller, Volume, src)
         VALUES
-        ('{products[i]}', '{prices[i]}', '{diagonals[i]}', '{resolutions[i]}', '{CPUs[i]}', '{RAMs[i]}', '{Graphics_controllers[i]}', '{Volumes[i]}', '{filename}');"""
+        ('{productst[i].text.strip()}', '{t.strip()}', '{u[0][:u[0].find("/")]}', '{u[0][u[0].find("/")+1:]}', 
+        '{u[2]}', '{u[4]+" "+u[5]}', '{u[6]}', '{u[8]}', '{filename}');"""
     cursor.execute(insert)
     connection.commit()
 cursor.execute("select * from laptops")
 print("Результат", cursor.fetchall())
 cursor.close()
 connection.close()
-

Задания/task1/Lahin/postgresEasy.py

@@ -1,42 +0,0 @@
-import psycopg2
-import wget
-from bs4 import BeautifulSoup
-from selenium import webdriver
-from selenium.webdriver.chrome.service import Service
-s = Service("С:\DATA\ChromeDriver\chromedriver.exe")
-browser = webdriver.Chrome(service=s)
-browser.get("https://online.metro-cc.ru/virtual/assortiment_rioba-4887?from=under_search&page=2")
-html_text = browser.page_source
-soup = BeautifulSoup(html_text, 'lxml')
-productst = soup.find_all('span', class_="product-card-name__text")
-pricest = soup.find_all('span', class_="product-price__sum-rubles")
-pictures = soup.find_all('img', class_="product-card-photo__image")
-connection=psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
-cursor = connection.cursor()
-products, prices = [], []
-for i in range(len(productst)):
-    products.append(productst[i].text.replace("\n", "").strip())
-    prices.append(pricest[i].text+" Р/шт")
-insert = """CREATE TABLE public.Chocolate(
-id serial primary key,
-product varchar(100),
-price varchar(25),
-src varchar(50)
-);
-"""
-cursor.execute(insert)
-connection.commit()
-for i in range(len(products)):
-    url = pictures[i]['src']
-    filename = f"C:\DATA\Images2\{i+1}.jpg"
-    wget.download(url, filename)
-    insert = f"""INSERT INTO public.Chocolate(
-        product, price, src)
-        VALUES
-        ('{products[i]}', '{prices[i]}', '{filename}');"""
-    cursor.execute(insert)
-    connection.commit()
-cursor.execute("select * from Chocolate")
-print("Результат", cursor.fetchall())
-cursor.close()
-connection.close()

Задания/task1/Markovich/Parsing

@@ -0,0 +1,40 @@
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+import wget
+import psycopg2
+
+connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
+cursor = connection.cursor()
+
+s = Service(r"C:\\Documents\\data\\chromedriver.exe")
+browser = webdriver.Chrome(service=s)
+browser.get('https://www.citilink.ru/catalog/smartfony/')
+html_text = browser.page_source
+b_soup = BeautifulSoup(html_text, 'lxml')
+
+insert_qwery = """create table phones\n
+(id serial primary key, name varchar(100), price varchar(100),delivery varchar(100), pictures varchar(100))"""
+
+cursor.execute(insert_qwery)
+connection.commit()
+
+phone_names = b_soup.find_all('a', class_="app-catalog-9gnskf e1259i3g0")
+phone_price = b_soup.find_all('span', class_="e1j9birj0 e106ikdt0 app-catalog-j8h82j e1gjr6xo0")
+phone_delivery = b_soup.find_all('div', class_="e16h9msl0 app-catalog-1fnzrtt e15ufesp0")
+pictures = b_soup.find_all('div', class_="app-catalog-zugobn e1o1y6qd0")
+
+for i in range(15):
+    url = 'https://www.citilink.ru/catalog/smartfony/'
+    filename = f"C:\\Users\\mark6\\OneDrive\\Desktop\\БД\\img\\{i}.jpg"
+    wget.download(url, filename)
+    ins_qwery = f"""INSERT INTO public.phones(name, price, delivery, pictures)\n
+    VALUES ('{phone_names[i].text}', '{phone_price[i].text}', '{phone_delivery[i].text}', '{filename}');"""
+    cursor.execute(ins_qwery)
+    connection.commit()
+
+cursor.execute("Select * from phones")
+print(cursor.fetchall())
+
+cursor.close()
+connection.close()

Задания/task1/Uskova/DataBaze.py

@@ -0,0 +1,39 @@
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+import psycopg2
+import wget
+
+connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4')
+cursor = connection.cursor()
+
+crtable = """ create table Notebooks
+    (id serial primary key, name varchar(500), description varchar(500), code varchar(200), price varchar(100),
+    picture varchar(500)) """
+
+cursor.execute(crtable)
+connection.commit()
+
+s = Service('C:\\Users\\Yekaterina\\Downloads\\chromedriver_win32\\chromedriver.exe')
+browser = webdriver.Chrome(service=s)
+browser.get('https://www.citilink.ru/catalog/noutbuki/')
+html_text = browser.page_source
+soup = BeautifulSoup(html_text, 'lxml')
+
+name = soup.find_all('div', class_='app-catalog-1tp0ino e1an64qs0')
+description = soup.find_all('div', class_='app-catalog-1o4umte eevw8x70')
+code = soup.find_all('div', class_= 'app-catalog-0 e1dsj6g20')
+price = soup.find_all('div', class_= 'app-catalog-0 e1dsj6g20')
+picture = soup.find_all('div', class_='app-catalog-0 e1jarwcz0')
+
+for i in range(10):
+    url = picture[i].find('img').attrs['src']
+    file = f"C:\\Users\\Yekaterina\\Desktop\\baza\\pic{i}.JPG"
+    wget.download(url, file)
+    ins_qwery = f"""insert into public.Notebooks(name, description, code, price, picture)
+    values ('{name[i].text}', '{description[i].text}', '{code[i].text}', '{price[i].text}','{file}')"""
+    cursor.execute(ins_qwery)
+    connection.commit()
+
+cursor.close()
+connection.close()

Задания/task1/Yaroshevskiy/main.py

@@ -21,8 +21,11 @@ with psyc.connect(dbname="db_for_parse", user="perfecto") as conn:
             img_tag = product.find(attrs={"class": "imagef"}).find("img")
             title = img_tag.attrs.get("alt")
             image = url + img_tag.attrs.get("src")
+            price = product.find(attrs={"class": "product-price"}).text
+            desc = product.find(attrs={"class": "product-desc"}).text
             download(url + img_tag.attrs.get("src"), f"images/{i}.jpg")
 
-            cursor.execute(f"insert into images(link, name) values ('{image}', '{title}')")
+            cursor.execute(f"""insert into images(link, name, price, description, file)
+                               values ('{image}', '{title}', '{price}', '{desc}', 'images/{i}.jpg')""")
         
         conn.commit()