Merge branch 'features/SU2-2' of DAAntipenko/finreport-analyzer into develop

Gemfile

@@ -6,6 +6,10 @@ ruby "3.3.0"
 # Bundle edge Rails instead: gem "rails", github: "rails/rails", branch: "main"
 gem "rails", "~> 7.1.3", ">= 7.1.3.2"
 
+gem "sidekiq" # background jobs processing
+gem 'httparty' # for http requests
+gem 'rubyzip' # for zip file processing
+
 # The original asset pipeline for Rails [https://github.com/rails/sprockets-rails]
 gem "sprockets-rails"
 
@@ -60,7 +64,7 @@ gem 'inline_svg' # inline svg in html
 gem 'rack-cas', github: 'aleksandrov1988/rack-cas', branch: 'master'
 gem 'addressable', github: 'aleksandrov1988/addressable', branch: 'master'
 
-gem "pagy" # for turbo pagination 
+gem "pagy" # for turbo pagination
 
 group :development, :test do
   # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem

Gemfile.lock

@@ -143,6 +143,7 @@ GEM
     concurrent-ruby (1.3.4)
     connection_pool (2.4.1)
     crass (1.0.6)
+    csv (3.3.5)
     dartsass-sprockets (3.1.0)
       railties (>= 4.0.0)
       sassc-embedded (~> 1.69)
@@ -191,6 +192,10 @@ GEM
       actioncable (>= 6.0.0)
       listen (>= 3.0.0)
       railties (>= 6.0.0)
+    httparty (0.23.2)
+      csv
+      mini_mime (>= 1.0.0)
+      multi_xml (>= 0.5.2)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
     importmap-rails (2.0.3)
@@ -232,6 +237,8 @@ GEM
     mini_mime (1.1.5)
     minitest (5.25.1)
     msgpack (1.7.5)
+    multi_xml (0.7.2)
+      bigdecimal (~> 3.1)
     mutex_m (0.3.0)
     net-imap (0.5.1)
       date
@@ -375,6 +382,12 @@ GEM
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
+    sidekiq (7.3.9)
+      base64
+      connection_pool (>= 2.3.0)
+      logger
+      rack (>= 2.2.4)
+      redis-client (>= 0.22.2)
     slim (5.2.1)
       temple (~> 0.10.0)
       tilt (>= 2.1.0)
@@ -467,6 +480,7 @@ DEPENDENCIES
   debug
   foreman
   hotwire-livereload (>= 1.3.0)
+  httparty
   importmap-rails
   inline_svg
   inputmask-rails
@@ -483,7 +497,9 @@ DEPENDENCIES
   rails (~> 7.1.3, >= 7.1.3.2)
   redis (>= 4.0.1)
   rspec-rails (~> 6.1.0)
+  rubyzip
   selenium-webdriver
+  sidekiq
   slim-rails
   sprockets-rails
   stimulus-rails

Procfile.dev

@@ -1,2 +1,3 @@
-web: env RUBY_DEBUG_OPEN=true bin/rails server
+web: env RUBY_DEBUG_OPEN=true bin/rails server -p 3000
 css: bin/rails tailwindcss:watch
+worker: bundle exec sidekiq

app/controllers/reports_controller.rb

@@ -0,0 +1,14 @@
+class ReportsController < ApplicationController
+  def index
+  end
+
+  def create
+    inn = params[:inn]
+    output_path = Rails.root.join('tmp', 'reports', inn)
+    FileUtils.mkdir_p(output_path)
+
+    FileProcessorJob.perform_async(inn, output_path.to_s)
+
+    redirect_to reports_path, notice: 'Отчёт загружается в фоне'
+  end
+end

app/helpers/reports_helper.rb

@@ -0,0 +1,2 @@
+module ReportsHelper
+end

app/sidekiq/file_processor_job.rb

@@ -0,0 +1,53 @@
+require 'zip'
+require 'stringio'
+
+
+class FileProcessorJob
+  include Sidekiq::Job
+
+  def perform(inn, output_path)
+    Rails.logger.info "=== Начало загрузки для ИНН: #{inn} ==="
+    download_inn(inn, output_path)
+    Rails.logger.info "=== Завершено для ИНН: #{inn} ==="
+  rescue => e
+    Rails.logger.error "=== ОШИБКА: #{e.message} ==="
+    Rails.logger.error e.backtrace.first(5).join("\n")
+    raise
+  end
+
+  private
+
+  def get_content(url, json_need = false)
+    headers = {
+      'User-Agent' => "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
+      'Accept-Language' => 'en-US,en;q=0.9',
+      'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+    }
+    content = HTTParty.get(url, headers: headers).body
+    json_need ? JSON.parse(content) : content
+  end
+
+  def download_inn(inn, output)
+    url = "https://bo.nalog.gov.ru/advanced-search/organizations/search?query=#{inn}&page=0&size=20"
+    content = get_content(url, true)
+    id = content['content'][0]['id']
+
+    url = "https://bo.nalog.gov.ru/nbo/organizations/#{id}/bfo/"
+    content = get_content(url, true)
+    periods_and_details_ids = content.map { |item| [item['period'], item['typeCorrections'][0]['correction']['id']] }
+
+    periods_and_details_ids.each do |period, details_id|
+      url = "https://bo.nalog.gov.ru/download/bfo/#{id}?auditReport=false&balance=true"\
+            "&capitalChange=true&clarification=false&targetedFundsUsing=false&detailsId=#{details_id}"\
+            "&financialResult=true&fundsMovement=true&type=XLS&period=#{period}"
+      content = get_content(url)
+
+      zip_file = Zip::File.open_buffer(StringIO.new(content))
+      zip_file.each do |entry|
+        output_file = File.join(output, entry.name)
+        FileUtils.mkdir_p(File.dirname(output_file))
+        zip_file.extract(entry, output_file) { true } # перезаписывать если существует
+      end
+    end
+  end
+end

app/views/reports/create.html.erb

@@ -0,0 +1,4 @@
+<div>
+  <h1 class="font-bold text-4xl">Reports#create</h1>
+  <p>Find me in app/views/reports/create.html.erb</p>
+</div>

app/views/reports/index.html.erb

@@ -0,0 +1,17 @@
+<!-- app/views/reports/index.html.erb -->
+<div style="max-width: 500px; margin: 50px auto; padding: 20px;">
+  <h1>Загрузка отчётов по ИНН</h1>
+
+  <% if notice %>
+    <p style="color: green;"><%= notice %></p>
+  <% end %>
+
+  <%= form_with url: reports_path, method: :post, local: true do |f| %>
+    <div style="margin-bottom: 15px;">
+      <%= f.label :inn, "Введите ИНН:" %>
+      <%= f.text_field :inn, placeholder: "1234567890", required: true, style: "width: 100%; padding: 8px;" %>
+    </div>
+
+    <%= f.submit "Загрузить отчёт", style: "padding: 10px 20px; cursor: pointer;" %>
+  <% end %>
+</div>

config/initializers/sidekiq.rb

@@ -0,0 +1,7 @@
+Sidekiq.configure_server do |config|
+  config.redis = { url: 'redis://localhost:6379/0' }
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = { url: 'redis://localhost:6379/0' }
+end

config/routes.rb

@@ -1,5 +1,12 @@
+# require 'sidekiq/web'
+
 Rails.application.routes.draw do
-  # Define your application routes per the DSL in https://guides.rubyonrails.org/routing.html
+  get 'reports/create'
+  get '/reports', to: 'reports#index'
+  post '/reports', to: 'reports#create'
+
+  # mount Sidekiq::Web => '/sidekiq' # Для проверки работы Sidekiq через веб-интерфейс
+
 
   # Defines the root path route ("/")
   root "welcome#home"

lib/scripts/script.rb

@@ -0,0 +1,45 @@
+require 'httparty'
+require 'json'
+require 'zip'
+require 'stringio'
+require 'optparse'
+
+def get_content(url, json_need = false)
+  headers = {
+      'User-Agent' => "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
+      'Accept-Language' => 'en-US,en;q=0.9',
+      'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
+  }
+  content = HTTParty.get(url, headers: headers).body
+  json_need ? JSON.parse(content) : content
+end
+
+def download_INN(inn, output)
+  url = "https://bo.nalog.gov.ru/advanced-search/organizations/search?query=#{inn}&page=0&size=20"
+  content = get_content(url, true)
+  id = content['content'][0]['id']
+  url = "https://bo.nalog.gov.ru/nbo/organizations/#{id}/bfo/"
+  content = get_content(url, true)
+  periods_and_details_ids = content.map { |item| [item['period'], item['typeCorrections'][0]['correction']['id']] }
+  periods_and_details_ids.each do |period, details_id|
+    url = "https://bo.nalog.gov.ru/download/bfo/#{id}?auditReport=false&balance=true"\
+    "&capitalChange=true&clarification=false&targetedFundsUsing=false&detailsId=#{details_id}"\
+    "&financialResult=true&fundsMovement=true&type=XLS&period=#{period}"
+    content = get_content(url)
+    zip_file = Zip::File.open_buffer(StringIO.new(content))
+    zip_file.each do |entry|
+      zip_file.extract(entry, destination_directory: output)
+    end
+  end
+end
+
+def main
+  options = {}
+  OptionParser.new do |opt|
+    opt.on('--inn INN', 'INN') { |o| options[:INN] = o }
+    opt.on('--output OUTPUT', 'OUTPUT') { |o| options[:output_path] = o }
+  end.parse!
+  download_INN(options[:INN], options[:output_path])
+end
+
+main

spec/helpers/reports_helper_spec.rb

@@ -0,0 +1,15 @@
+require 'rails_helper'
+
+# Specs in this file have access to a helper object that includes
+# the ReportsHelper. For example:
+#
+# describe ReportsHelper do
+#   describe "string concat" do
+#     it "concats two strings with spaces" do
+#       expect(helper.concat_strings("this","that")).to eq("this that")
+#     end
+#   end
+# end
+RSpec.describe ReportsHelper, type: :helper do
+  pending "add some examples to (or delete) #{__FILE__}"
+end

spec/requests/reports_spec.rb

@@ -0,0 +1,11 @@
+require 'rails_helper'
+
+RSpec.describe "Reports", type: :request do
+  describe "GET /create" do
+    it "returns http success" do
+      get "/reports/create"
+      expect(response).to have_http_status(:success)
+    end
+  end
+
+end

spec/sidekiq/file_processor_job_spec.rb

@@ -0,0 +1,4 @@
+require 'rails_helper'
+RSpec.describe FileProcessorJob, type: :job do
+  pending "add some examples to (or delete) #{__FILE__}"
+end

spec/views/reports/create.html.tailwindcss_spec.rb

@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe "reports/create.html.tailwindcss", type: :view do
+  pending "add some examples to (or delete) #{__FILE__}"
+end