diff --git a/Gemfile b/Gemfile index 578374f..7eb2513 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,10 @@ ruby "3.3.0" # Bundle edge Rails instead: gem "rails", github: "rails/rails", branch: "main" gem "rails", "~> 7.1.3", ">= 7.1.3.2" +gem "sidekiq" # background jobs processing +gem 'httparty' # for http requests +gem 'rubyzip' # for zip file processing + # The original asset pipeline for Rails [https://github.com/rails/sprockets-rails] gem "sprockets-rails" @@ -60,7 +64,7 @@ gem 'inline_svg' # inline svg in html gem 'rack-cas', github: 'aleksandrov1988/rack-cas', branch: 'master' gem 'addressable', github: 'aleksandrov1988/addressable', branch: 'master' -gem "pagy" # for turbo pagination +gem "pagy" # for turbo pagination group :development, :test do # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem diff --git a/Gemfile.lock b/Gemfile.lock index b6cc204..866345b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -143,6 +143,7 @@ GEM concurrent-ruby (1.3.4) connection_pool (2.4.1) crass (1.0.6) + csv (3.3.5) dartsass-sprockets (3.1.0) railties (>= 4.0.0) sassc-embedded (~> 1.69) @@ -191,6 +192,10 @@ GEM actioncable (>= 6.0.0) listen (>= 3.0.0) railties (>= 6.0.0) + httparty (0.23.2) + csv + mini_mime (>= 1.0.0) + multi_xml (>= 0.5.2) i18n (1.14.6) concurrent-ruby (~> 1.0) importmap-rails (2.0.3) @@ -232,6 +237,8 @@ GEM mini_mime (1.1.5) minitest (5.25.1) msgpack (1.7.5) + multi_xml (0.7.2) + bigdecimal (~> 3.1) mutex_m (0.3.0) net-imap (0.5.1) date @@ -375,6 +382,12 @@ GEM rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) + sidekiq (7.3.9) + base64 + connection_pool (>= 2.3.0) + logger + rack (>= 2.2.4) + redis-client (>= 0.22.2) slim (5.2.1) temple (~> 0.10.0) tilt (>= 2.1.0) @@ -467,6 +480,7 @@ DEPENDENCIES debug foreman hotwire-livereload (>= 1.3.0) + httparty importmap-rails inline_svg inputmask-rails @@ -483,7 +497,9 @@ DEPENDENCIES rails (~> 7.1.3, >= 7.1.3.2) redis (>= 4.0.1) rspec-rails (~> 6.1.0) + rubyzip selenium-webdriver + sidekiq slim-rails sprockets-rails stimulus-rails diff --git a/Procfile.dev b/Procfile.dev index cce5ded..f084d6a 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -1,2 +1,3 @@ -web: env RUBY_DEBUG_OPEN=true bin/rails server +web: env RUBY_DEBUG_OPEN=true bin/rails server -p 3000 css: bin/rails tailwindcss:watch +worker: bundle exec sidekiq \ No newline at end of file diff --git a/app/controllers/reports_controller.rb b/app/controllers/reports_controller.rb new file mode 100644 index 0000000..9d76023 --- /dev/null +++ b/app/controllers/reports_controller.rb @@ -0,0 +1,14 @@ +class ReportsController < ApplicationController + def index + end + + def create + inn = params[:inn] + output_path = Rails.root.join('tmp', 'reports', inn) + FileUtils.mkdir_p(output_path) + + FileProcessorJob.perform_async(inn, output_path.to_s) + + redirect_to reports_path, notice: 'Отчёт загружается в фоне' + end +end \ No newline at end of file diff --git a/app/helpers/reports_helper.rb b/app/helpers/reports_helper.rb new file mode 100644 index 0000000..cae2f09 --- /dev/null +++ b/app/helpers/reports_helper.rb @@ -0,0 +1,2 @@ +module ReportsHelper +end diff --git a/app/sidekiq/file_processor_job.rb b/app/sidekiq/file_processor_job.rb new file mode 100644 index 0000000..8f36cec --- /dev/null +++ b/app/sidekiq/file_processor_job.rb @@ -0,0 +1,53 @@ +require 'zip' +require 'stringio' + + +class FileProcessorJob + include Sidekiq::Job + + def perform(inn, output_path) + Rails.logger.info "=== Начало загрузки для ИНН: #{inn} ===" + download_inn(inn, output_path) + Rails.logger.info "=== Завершено для ИНН: #{inn} ===" + rescue => e + Rails.logger.error "=== ОШИБКА: #{e.message} ===" + Rails.logger.error e.backtrace.first(5).join("\n") + raise + end + + private + + def get_content(url, json_need = false) + headers = { + 'User-Agent' => "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + 'Accept-Language' => 'en-US,en;q=0.9', + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' + } + content = HTTParty.get(url, headers: headers).body + json_need ? JSON.parse(content) : content + end + + def download_inn(inn, output) + url = "https://bo.nalog.gov.ru/advanced-search/organizations/search?query=#{inn}&page=0&size=20" + content = get_content(url, true) + id = content['content'][0]['id'] + + url = "https://bo.nalog.gov.ru/nbo/organizations/#{id}/bfo/" + content = get_content(url, true) + periods_and_details_ids = content.map { |item| [item['period'], item['typeCorrections'][0]['correction']['id']] } + + periods_and_details_ids.each do |period, details_id| + url = "https://bo.nalog.gov.ru/download/bfo/#{id}?auditReport=false&balance=true"\ + "&capitalChange=true&clarification=false&targetedFundsUsing=false&detailsId=#{details_id}"\ + "&financialResult=true&fundsMovement=true&type=XLS&period=#{period}" + content = get_content(url) + + zip_file = Zip::File.open_buffer(StringIO.new(content)) + zip_file.each do |entry| + output_file = File.join(output, entry.name) + FileUtils.mkdir_p(File.dirname(output_file)) + zip_file.extract(entry, output_file) { true } # перезаписывать если существует + end + end + end +end \ No newline at end of file diff --git a/app/views/reports/create.html.erb b/app/views/reports/create.html.erb new file mode 100644 index 0000000..cb87c0b --- /dev/null +++ b/app/views/reports/create.html.erb @@ -0,0 +1,4 @@ +
+

Reports#create

+

Find me in app/views/reports/create.html.erb

+
diff --git a/app/views/reports/index.html.erb b/app/views/reports/index.html.erb new file mode 100644 index 0000000..ea83847 --- /dev/null +++ b/app/views/reports/index.html.erb @@ -0,0 +1,17 @@ + +
+

Загрузка отчётов по ИНН

+ + <% if notice %> +

<%= notice %>

+ <% end %> + + <%= form_with url: reports_path, method: :post, local: true do |f| %> +
+ <%= f.label :inn, "Введите ИНН:" %> + <%= f.text_field :inn, placeholder: "1234567890", required: true, style: "width: 100%; padding: 8px;" %> +
+ + <%= f.submit "Загрузить отчёт", style: "padding: 10px 20px; cursor: pointer;" %> + <% end %> +
\ No newline at end of file diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb new file mode 100644 index 0000000..d6763f4 --- /dev/null +++ b/config/initializers/sidekiq.rb @@ -0,0 +1,7 @@ +Sidekiq.configure_server do |config| + config.redis = { url: 'redis://localhost:6379/0' } +end + +Sidekiq.configure_client do |config| + config.redis = { url: 'redis://localhost:6379/0' } +end \ No newline at end of file diff --git a/config/routes.rb b/config/routes.rb index b05bc9c..5d9a41b 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,12 @@ +# require 'sidekiq/web' + Rails.application.routes.draw do - # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html + get 'reports/create' + get '/reports', to: 'reports#index' + post '/reports', to: 'reports#create' + + # mount Sidekiq::Web => '/sidekiq' # Для проверки работы Sidekiq через веб-интерфейс + # Defines the root path route ("/") root "welcome#home" diff --git a/lib/scripts/script.rb b/lib/scripts/script.rb new file mode 100644 index 0000000..2de01e6 --- /dev/null +++ b/lib/scripts/script.rb @@ -0,0 +1,45 @@ +require 'httparty' +require 'json' +require 'zip' +require 'stringio' +require 'optparse' + +def get_content(url, json_need = false) + headers = { + 'User-Agent' => "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + 'Accept-Language' => 'en-US,en;q=0.9', + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8' + } + content = HTTParty.get(url, headers: headers).body + json_need ? JSON.parse(content) : content +end + +def download_INN(inn, output) + url = "https://bo.nalog.gov.ru/advanced-search/organizations/search?query=#{inn}&page=0&size=20" + content = get_content(url, true) + id = content['content'][0]['id'] + url = "https://bo.nalog.gov.ru/nbo/organizations/#{id}/bfo/" + content = get_content(url, true) + periods_and_details_ids = content.map { |item| [item['period'], item['typeCorrections'][0]['correction']['id']] } + periods_and_details_ids.each do |period, details_id| + url = "https://bo.nalog.gov.ru/download/bfo/#{id}?auditReport=false&balance=true"\ + "&capitalChange=true&clarification=false&targetedFundsUsing=false&detailsId=#{details_id}"\ + "&financialResult=true&fundsMovement=true&type=XLS&period=#{period}" + content = get_content(url) + zip_file = Zip::File.open_buffer(StringIO.new(content)) + zip_file.each do |entry| + zip_file.extract(entry, destination_directory: output) + end + end +end + +def main + options = {} + OptionParser.new do |opt| + opt.on('--inn INN', 'INN') { |o| options[:INN] = o } + opt.on('--output OUTPUT', 'OUTPUT') { |o| options[:output_path] = o } + end.parse! + download_INN(options[:INN], options[:output_path]) +end + +main \ No newline at end of file diff --git a/spec/helpers/reports_helper_spec.rb b/spec/helpers/reports_helper_spec.rb new file mode 100644 index 0000000..258e004 --- /dev/null +++ b/spec/helpers/reports_helper_spec.rb @@ -0,0 +1,15 @@ +require 'rails_helper' + +# Specs in this file have access to a helper object that includes +# the ReportsHelper. For example: +# +# describe ReportsHelper do +# describe "string concat" do +# it "concats two strings with spaces" do +# expect(helper.concat_strings("this","that")).to eq("this that") +# end +# end +# end +RSpec.describe ReportsHelper, type: :helper do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/requests/reports_spec.rb b/spec/requests/reports_spec.rb new file mode 100644 index 0000000..9ff0a1a --- /dev/null +++ b/spec/requests/reports_spec.rb @@ -0,0 +1,11 @@ +require 'rails_helper' + +RSpec.describe "Reports", type: :request do + describe "GET /create" do + it "returns http success" do + get "/reports/create" + expect(response).to have_http_status(:success) + end + end + +end diff --git a/spec/sidekiq/file_processor_job_spec.rb b/spec/sidekiq/file_processor_job_spec.rb new file mode 100644 index 0000000..01e9f7c --- /dev/null +++ b/spec/sidekiq/file_processor_job_spec.rb @@ -0,0 +1,4 @@ +require 'rails_helper' +RSpec.describe FileProcessorJob, type: :job do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/views/reports/create.html.tailwindcss_spec.rb b/spec/views/reports/create.html.tailwindcss_spec.rb new file mode 100644 index 0000000..f72bcdd --- /dev/null +++ b/spec/views/reports/create.html.tailwindcss_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe "reports/create.html.tailwindcss", type: :view do + pending "add some examples to (or delete) #{__FILE__}" +end