diff --git a/Задания/task1/Epifanov/FHWDB.py b/Задания/task1/Epifanov/FHWDB.py index 75f4a5b..7a06660 100644 --- a/Задания/task1/Epifanov/FHWDB.py +++ b/Задания/task1/Epifanov/FHWDB.py @@ -5,28 +5,30 @@ import psycopg2 import wget browser = webdriver.Chrome(service=Service('C:\Desktop\exe\chromedriver.exe')) -browser.get('https://flawery.ru/moscow/bouquets/event-yanvary25/?min=0&max=max&sorting=discount&filter_24h=on') +browser.get('https://flawery.ru/moscow/bouquets/color-blue/?min=0&max=max&sorting=discount') soup = BeautifulSoup(browser.page_source, "lxml") Name = soup.find_all(attrs={"class": "catalog_title"}) Price = soup.find_all(attrs={"class": "catalog_price_now"}) -Delivery_Time = soup.find_all(attrs={"class": "catalog_express"}) Discount = soup.find_all(attrs={"class": "catalog_sale"}) Image = soup.find_all(attrs={"class": "catalog_item catalog_item_popup"}) connection = psycopg2.connect(host='localhost', dbname='FHWDB', user='postgres', password='Q1w2e3r4') cursor = connection.cursor() create_q = '''CREATE TABLE Parse - (ID serial primary key, Name varchar(100), Price varchar(9), Delivery_Time varchar(25), Discount varchar(10), src varchar(110))''' + (ID serial primary key, Name varchar(100), Price varchar(9), Discount varchar(10), src_of_picture varchar(50))''' cursor.execute(create_q) connection.commit() for j in range(60): url = 'https://flawery.ru'+Image[j].find('a').find('picture').find('img').attrs['src'] - tempf = f"C:\\Users\\user\\Desktop\\FHWDB{j}.jpg" + tempf = f"C:\\Users\\user\\PFDB{j}.jpg" wget.download(url, tempf) - insert_query = f'''INSERT into public.Parse(Name, Price, Delivery_Time, Discount, src) values ('{Name[j].text}', '{Price[j].text}', '{Delivery_Time[j].text}', '{Discount[j].text}', '{tempf}') ''' + insert_query = f'''INSERT into public.Parse(Name, Price, Discount, src_of_picture) values ('{Name[j].text}', '{Price[j].text}', '{Discount[j].text}', '{tempf}') ''' cursor.execute(insert_query) - connection.commit() + +cursor.execute("SELECT * from Parse") +print(cursor.fetchall()) + cursor.close() connection.close() diff --git a/Задания/task1/Garanyan/main.py b/Задания/task1/Garanyan/main.py index ac6f8f2..8e6da15 100644 --- a/Задания/task1/Garanyan/main.py +++ b/Задания/task1/Garanyan/main.py @@ -1,12 +1,35 @@ from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.chrome.service import Service +import psycopg2 +import wget -s = Service("E:\data\chromedriver.exe") -browser = webdriver.Chrome(service=s) -browser.get("https://ria.ru/") -html_text = browser.page_source -soup = BeautifulSoup(html_text, 'lxml') -news = soup.find_all('div', class_='cell cell-list') -for new in news: - print(new.text) +browser = webdriver.Chrome(service=Service('E:\data\chromedriver.exe')) +browser.get('https://8956food.ru/') +soup = BeautifulSoup(browser.page_source, "lxml") + +Name = soup.find_all(attrs={"class": "subtitle"}) +Price = soup.find_all(attrs={"class": "price"}) +Image = soup.find_all(attrs={"class": "holder-img"}) + +connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4') +cursor = connection.cursor() +create_q = '''CREATE TABLE Food + (ID serial primary key, Name varchar(500), Price varchar(60), src varchar(100))''' +cursor.execute(create_q) +connection.commit() + +#https://smartomato.ams3.cdn.digitaloceanspaces.com/uploads/media/photo/769221/dish_large__1.jpg +for j in range(15): + url = Image[j].find('img').attrs['src'] + tempf = f"C:\\Users\\Harutyun\\Desktop\\prog\\food{j}.jpg" + wget.download(url, tempf) + insert_query = f'''INSERT into public.Food(Name, Price, src) values ('{Name[j].text}', '{Price[j].text}', '{tempf}') ''' + cursor.execute(insert_query) +connection.commit() + +cursor.execute("SELECT * from Food") +print(cursor.fetchall()) + +cursor.close() +connection.close() \ No newline at end of file diff --git a/Задания/task1/Kluchinskaya1/parsing_2.py b/Задания/task1/Kluchinskaya1/parsing_2.py new file mode 100644 index 0000000..df08832 --- /dev/null +++ b/Задания/task1/Kluchinskaya1/parsing_2.py @@ -0,0 +1,29 @@ +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +from wget import download +import psycopg2 +connection = psycopg2.connect(host='localhost', dbname='dbdata', + user='postgres', password='Q1w2e3r4') +cursor=connection.cursor() +s = Service('C:\\Users\\Dasha\\Desktop\\Progect\\chromedriver.exe') +browser = webdriver.Chrome(service=s) +browser = webdriver.Chrome() +url = 'https://www.tretyakovgallery.ru/' +browser.get(url) +html_text=browser.page_source +soup = BeautifulSoup(html_text, 'lxml') +products = soup.find_all(attrs={"class": "building-item"}) +for i, product in enumerate(products): + link = product.find("img").attrs.get("src") + tag = product.find("strong") + name = product.find(attrs={"class": "building-item__title"}) + description = product.find(attrs={"class": "building-item__description"}) + if tag is not None: + download(url + link, f"img\\{i}.jpg") + cursor.execute(f"""INSERT INTO public.parsing(url, workhours, name, description, filename) + VALUES + ( '{url + link}', '{tag.text}', '{name.text.strip()}', '{description.text.strip()}', 'img\\{i}.jpg')""") +connection.commit() +cursor.close() +connection.close() \ No newline at end of file diff --git a/Задания/task1/Lahin/postgres.py b/Задания/task1/Lahin/postgres.py index 0ca1bb2..cfa0920 100644 --- a/Задания/task1/Lahin/postgres.py +++ b/Задания/task1/Lahin/postgres.py @@ -7,65 +7,55 @@ import time i = 1 s = Service("С:\DATA\ChromeDriver\chromedriver.exe") browser = webdriver.Chrome(service=s) -browser.get("https://www.mvideo.ru/noutbuki-planshety-komputery-8/noutbuki-118?from=under_search") +browser.get("https://www.mvideo.ru/noutbuki-planshety-komputery-8/noutbuki-118/f/tolko-v-nalichii=da?from=under_search") +time.sleep(5) +""" height = browser.execute_script("return document.body.scrollHeight") time.sleep(4) -while 900*i < height: - height = browser.execute_script("return document.body.scrollHeight") - browser.execute_script(f"window.scrollTo(0,{900*i})") +for i in range(10): + browser.execute_script(f"window.scrollTo(0, {1000*i})") time.sleep(1) - i+=1 +""" html_text = browser.page_source soup = BeautifulSoup(html_text, 'lxml') productst = soup.find_all('a', class_="product-title__text") pricest = soup.find_all('span', class_="price__main-value") -features = soup.find_all('ul', class_="product-feature-list product-feature-list--undefined") -pictures = soup.find_all('img', class_="product-picture__img product-picture__img--list") -products, prices, diagonals, resolutions, CPUs, RAMs, Graphics_controllers, Volumes = [],[],[],[],[],[],[],[] -for i in range(len(productst)): - products.append(productst[i].text.strip()) - prices.append(pricest[i].text.replace("\xa0", "").strip()) -for feature in features: - u = feature.text - diagonals.append(u[20:u.find("\"",20)+1]) - resolutions.append(u[u.find("\"",20)+2:u.find(".", 23)+1]) - CPUs.append(u[u.find("Процессор",20)+9:u.find("ГГц",20)+3].replace("\xa0","")) - RAMs.append(" ".join(u[u.find("(RAM)",20)+5:u.find("Графический контроллер")].split())) - Graphics_controllers.append(u[u.find("Графический контроллер")+22:u.rfind("Объем")-2]) - Volumes.append(u[u.rfind("Объем")+6:].replace("\xa0", "").strip()) -for i in range(len(Volumes)): - u = Volumes[i] - if u.rfind("D")!=-1: Volumes[i] = u[:u.rfind("D")+1]+ " " + u[u.rfind("D")+1:] - else: Volumes[i] = u[:u.rfind("С")+1]+ " " + u[u.rfind("С")+1:] +features = soup.find_all('span', class_="product-feature-list__value") +pictures = soup.find_all('div', class_="product-picture-container") connection=psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4') cursor = connection.cursor() -insert = """CREATE TABLE public.laptops( +""" insert = CREATE TABLE public.laptops( id serial primary key, product varchar(100), price varchar(15), diagonal varchar(5), resolution varchar(20), CPU varchar(50), -RAM varchar(10), +RAM varchar(15), Graphics_Controller varchar(40), Volume varchar(25), src varchar(100) ); """ +insert = """TRUNCATE TABLE public.laptops; ALTER SEQUENCE laptops_id_seq RESTART WITH 1;""" cursor.execute(insert) connection.commit() -for i in range(len(products)): - url = "https://"+pictures[i]['src'] +for i in range(len(productst)): + url = "https://"+pictures[i].find('img')['src'] filename = f"C:\DATA\Images\{i+1}.jpg" + u = [] + for j in range(10): + u.append(features[10*i+j].text) wget.download(url, filename) + t = pricest[i].text.replace("\xa0", " ") insert = f"""INSERT INTO public.laptops( product, price, diagonal, resolution, CPU, RAM, Graphics_Controller, Volume, src) VALUES - ('{products[i]}', '{prices[i]}', '{diagonals[i]}', '{resolutions[i]}', '{CPUs[i]}', '{RAMs[i]}', '{Graphics_controllers[i]}', '{Volumes[i]}', '{filename}');""" + ('{productst[i].text.strip()}', '{t.strip()}', '{u[0][:u[0].find("/")]}', '{u[0][u[0].find("/")+1:]}', + '{u[2]}', '{u[4]+" "+u[5]}', '{u[6]}', '{u[8]}', '{filename}');""" cursor.execute(insert) connection.commit() cursor.execute("select * from laptops") print("Результат", cursor.fetchall()) cursor.close() connection.close() - diff --git a/Задания/task1/Lahin/postgresEasy.py b/Задания/task1/Lahin/postgresEasy.py deleted file mode 100644 index 375b40d..0000000 --- a/Задания/task1/Lahin/postgresEasy.py +++ /dev/null @@ -1,42 +0,0 @@ -import psycopg2 -import wget -from bs4 import BeautifulSoup -from selenium import webdriver -from selenium.webdriver.chrome.service import Service -s = Service("С:\DATA\ChromeDriver\chromedriver.exe") -browser = webdriver.Chrome(service=s) -browser.get("https://online.metro-cc.ru/virtual/assortiment_rioba-4887?from=under_search&page=2") -html_text = browser.page_source -soup = BeautifulSoup(html_text, 'lxml') -productst = soup.find_all('span', class_="product-card-name__text") -pricest = soup.find_all('span', class_="product-price__sum-rubles") -pictures = soup.find_all('img', class_="product-card-photo__image") -connection=psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4') -cursor = connection.cursor() -products, prices = [], [] -for i in range(len(productst)): - products.append(productst[i].text.replace("\n", "").strip()) - prices.append(pricest[i].text+" Р/шт") -insert = """CREATE TABLE public.Chocolate( -id serial primary key, -product varchar(100), -price varchar(25), -src varchar(50) -); -""" -cursor.execute(insert) -connection.commit() -for i in range(len(products)): - url = pictures[i]['src'] - filename = f"C:\DATA\Images2\{i+1}.jpg" - wget.download(url, filename) - insert = f"""INSERT INTO public.Chocolate( - product, price, src) - VALUES - ('{products[i]}', '{prices[i]}', '{filename}');""" - cursor.execute(insert) - connection.commit() -cursor.execute("select * from Chocolate") -print("Результат", cursor.fetchall()) -cursor.close() -connection.close() diff --git a/Задания/task1/Markovich/Parsing b/Задания/task1/Markovich/Parsing new file mode 100644 index 0000000..62aaee7 --- /dev/null +++ b/Задания/task1/Markovich/Parsing @@ -0,0 +1,40 @@ +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +import wget +import psycopg2 + +connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4') +cursor = connection.cursor() + +s = Service(r"C:\\Documents\\data\\chromedriver.exe") +browser = webdriver.Chrome(service=s) +browser.get('https://www.citilink.ru/catalog/smartfony/') +html_text = browser.page_source +b_soup = BeautifulSoup(html_text, 'lxml') + +insert_qwery = """create table phones\n +(id serial primary key, name varchar(100), price varchar(100),delivery varchar(100), pictures varchar(100))""" + +cursor.execute(insert_qwery) +connection.commit() + +phone_names = b_soup.find_all('a', class_="app-catalog-9gnskf e1259i3g0") +phone_price = b_soup.find_all('span', class_="e1j9birj0 e106ikdt0 app-catalog-j8h82j e1gjr6xo0") +phone_delivery = b_soup.find_all('div', class_="e16h9msl0 app-catalog-1fnzrtt e15ufesp0") +pictures = b_soup.find_all('div', class_="app-catalog-zugobn e1o1y6qd0") + +for i in range(15): + url = 'https://www.citilink.ru/catalog/smartfony/' + filename = f"C:\\Users\\mark6\\OneDrive\\Desktop\\БД\\img\\{i}.jpg" + wget.download(url, filename) + ins_qwery = f"""INSERT INTO public.phones(name, price, delivery, pictures)\n + VALUES ('{phone_names[i].text}', '{phone_price[i].text}', '{phone_delivery[i].text}', '{filename}');""" + cursor.execute(ins_qwery) + connection.commit() + +cursor.execute("Select * from phones") +print(cursor.fetchall()) + +cursor.close() +connection.close() diff --git a/Задания/task1/Uskova/DataBaze.py b/Задания/task1/Uskova/DataBaze.py new file mode 100644 index 0000000..b3f2f60 --- /dev/null +++ b/Задания/task1/Uskova/DataBaze.py @@ -0,0 +1,39 @@ +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +import psycopg2 +import wget + +connection = psycopg2.connect(host='localhost', dbname='dbdata', user='postgres', password='Q1w2e3r4') +cursor = connection.cursor() + +crtable = """ create table Notebooks + (id serial primary key, name varchar(500), description varchar(500), code varchar(200), price varchar(100), + picture varchar(500)) """ + +cursor.execute(crtable) +connection.commit() + +s = Service('C:\\Users\\Yekaterina\\Downloads\\chromedriver_win32\\chromedriver.exe') +browser = webdriver.Chrome(service=s) +browser.get('https://www.citilink.ru/catalog/noutbuki/') +html_text = browser.page_source +soup = BeautifulSoup(html_text, 'lxml') + +name = soup.find_all('div', class_='app-catalog-1tp0ino e1an64qs0') +description = soup.find_all('div', class_='app-catalog-1o4umte eevw8x70') +code = soup.find_all('div', class_= 'app-catalog-0 e1dsj6g20') +price = soup.find_all('div', class_= 'app-catalog-0 e1dsj6g20') +picture = soup.find_all('div', class_='app-catalog-0 e1jarwcz0') + +for i in range(10): + url = picture[i].find('img').attrs['src'] + file = f"C:\\Users\\Yekaterina\\Desktop\\baza\\pic{i}.JPG" + wget.download(url, file) + ins_qwery = f"""insert into public.Notebooks(name, description, code, price, picture) + values ('{name[i].text}', '{description[i].text}', '{code[i].text}', '{price[i].text}','{file}')""" + cursor.execute(ins_qwery) + connection.commit() + +cursor.close() +connection.close() diff --git a/Задания/task1/Yaroshevskiy/main.py b/Задания/task1/Yaroshevskiy/main.py index 945ddf8..11eea2c 100644 --- a/Задания/task1/Yaroshevskiy/main.py +++ b/Задания/task1/Yaroshevskiy/main.py @@ -21,8 +21,11 @@ with psyc.connect(dbname="db_for_parse", user="perfecto") as conn: img_tag = product.find(attrs={"class": "imagef"}).find("img") title = img_tag.attrs.get("alt") image = url + img_tag.attrs.get("src") + price = product.find(attrs={"class": "product-price"}).text + desc = product.find(attrs={"class": "product-desc"}).text download(url + img_tag.attrs.get("src"), f"images/{i}.jpg") - cursor.execute(f"insert into images(link, name) values ('{image}', '{title}')") + cursor.execute(f"""insert into images(link, name, price, description, file) + values ('{image}', '{title}', '{price}', '{desc}', 'images/{i}.jpg')""") conn.commit()