Merge branch 'feature/SU2-19' of DAAntipenko/finreport-analyzer into develop

2026-03-25 19:20:28 +03:00
parent dcba938feb 7ee21d5e5d
commit 85fbac1cfd
9 changed files with 587 additions and 66 deletions
@@ -1,33 +1,21 @@
 require 'roo'

 class ReportParser
-  SHEET_NAME = 'Сведения об организации'
-  CELL_MAP = {
-    full_name: { row: 6,  col: 13 },
-    inn:       { row: 10, col: 13 },
-    kpp:       { row: 11, col: 13 },
-    okpo:      { row: 12, col: 13 },
-    okfs:      { row: 13, col: 13 },
-    okopf:     { row: 14, col: 13 },
-    okved2:    { row: 15, col: 13 },
-    address:   { row: 16, col: 13 }
-  }.freeze
-
-  AUDITOR_MAP = {
-    name: { row: 19, col: 13 },
-    inn:  { row: 20, col: 13 },
-    ogrn: { row: 21, col: 13 }
-  }.freeze
-
  def initialize(yearly_file)
    @yearly_file = yearly_file
  end

  def call
    ActiveRecord::Base.transaction do
-      parse_organization
-      parse_auditor
-      extract_year
+      ReportParsers::OrganizationInfoParser.new(@yearly_file, spreadsheet: spreadsheet).call
+
+      form_11_data = ReportParsers::Form11Parser.new(@yearly_file, spreadsheet: spreadsheet).call
+      form_12_data = ReportParsers::Form12Parser.new(@yearly_file, spreadsheet: spreadsheet).call
+      form_14_data = ReportParsers::Form14Parser.new(@yearly_file, spreadsheet: spreadsheet).call
+
+      save_form_statements(form_11_data, 1) # 1 - Бух. баланс
+      save_form_statements(form_12_data, 2) # 2 - Отчет о фин. результатах
+      save_form_statements(form_14_data, 4) # 4 - Отчет о движении денежных средств
    end
  end

@@ -43,56 +31,34 @@ class ReportParser
    end
  end

-  def info_sheet
-    @info_sheet ||= spreadsheet.sheet(SHEET_NAME)
-  end
+  def save_form_statements(form_data, form_id)
+    document_type = DocumentType.find_by(id: form_id)
+    raise "DocumentType with id=#{form_id} not found!" unless document_type

-  def read_cell(sheet, row, col)
-    value = sheet.cell(row, col)
-    value.is_a?(String) ? value.strip.presence : value
-  end
+    form_data.each do |year, data|
+      rows = data[:rows] || data['rows'] || []
+      next if rows.blank?

-  def parse_organization
-    org_data = CELL_MAP.each_with_object({}) do |(attr, pos), hash|
-      hash[attr] = read_cell(info_sheet, pos[:row], pos[:col])
+      statement = find_or_build_statement(document_type, year)
+      payload = {
+        data_year: year,
+        rows: rows
+      }
+      statement.assign_attributes(data: payload)
+      statement.save!
    end
-
-    organization = @yearly_file.organization
-    organization.update!(org_data)
-    organization
  end

-  def parse_auditor
-    auditor_data = AUDITOR_MAP.each_with_object({}) do |(attr, pos), hash|
-      hash[attr] = read_cell(info_sheet, pos[:row], pos[:col])
-    end
+  def find_or_build_statement(document_type, year)
+    existing = FinancialStatement
+               .joins(:yearly_file)
+               .where(document_type: document_type, sheet_number: year)
+               .where(yearly_files: { organization_id: @yearly_file.organization_id })
+               .order('yearly_files.year DESC NULLS LAST, financial_statements.updated_at DESC')
+               .first

-    return if auditor_data[:name].blank?
+    return existing if existing

-    auditor_inn = auditor_data[:inn].to_s.strip
-    auditor = if auditor_inn.present?
-                Auditor.find_or_initialize_by(inn: auditor_inn)
-              else
-                Auditor.find_or_initialize_by(name: auditor_data[:name])
-              end
-
-    auditor.assign_attributes(
-      name: auditor_data[:name],
-      inn: auditor_inn.presence,
-      ogrn: auditor_data[:ogrn].to_s.strip.presence
-    )
-    auditor.save!
-
-    @yearly_file.update!(auditor: auditor)
-    auditor
-  end
-
-  def extract_year
-    balance_sheet = spreadsheet.sheet('Бухгалтерский баланс')
-    period_text = balance_sheet.cell(4, 1).to_s
-    year = period_text.match(/(\d{4})/)&.captures&.first&.to_i
-
-    @yearly_file.update!(year: year) if year
-    year
+    @yearly_file.financial_statements.new(document_type: document_type, sheet_number: year)
  end
 end
@@ -0,0 +1,100 @@
+require 'set'
+
+module ReportParsers
+  class BaseParser
+    def initialize(yearly_file, spreadsheet:)
+      @yearly_file = yearly_file
+      @spreadsheet = spreadsheet
+    end
+
+    private
+
+    attr_reader :yearly_file, :spreadsheet
+
+    def sheet_exists?(sheet_name)
+      spreadsheet.sheets.include?(sheet_name)
+    end
+
+    def read_cell(sheet, row, col)
+      value = sheet.cell(row, col)
+
+      # Handle merged cells: if value is nil/blank, try reading from nearby cells
+      if value.blank? && sheet.respond_to?(:merged_cells)
+        value = find_merged_cell_value(sheet, row, col)
+      end
+
+      value.is_a?(String) ? value.strip.presence : value
+    end
+
+    def find_merged_cell_value(sheet, row, col)
+      # Try to find value in merged cell range
+      # Check cells in a small radius around the target cell
+      (-1..1).each do |row_offset|
+        (-1..1).each do |col_offset|
+          next if row_offset == 0 && col_offset == 0
+
+          check_row = row + row_offset
+          check_col = col + col_offset
+
+          next if check_row < 1 || check_col < 1
+          next if check_row > sheet.last_row || check_col > sheet.last_column
+
+          value = sheet.cell(check_row, check_col)
+          return value if value.present?
+        end
+      end
+
+      nil
+    end
+
+    def find_value_by_labels(sheet, labels, row_range:, value_offset: 1)
+      position = find_label_position(sheet, labels, row_range: row_range)
+      return if position.nil?
+
+      row, col = position
+
+      max_col = sheet.last_column.to_i
+      (col + value_offset).upto(max_col) do |search_col|
+        value = read_cell(sheet, row, search_col)
+        return value if value.present?
+      end
+
+      nil
+    end
+
+    def find_label_position(sheet, labels, row_range:)
+      normalized_labels = Array(labels).map { |label| normalize_label(label) }.to_set
+      max_col = sheet.last_column.to_i
+      return if max_col <= 1
+
+      row_range.each do |row|
+        1.upto(max_col - 1) do |col|
+          label_text = normalize_label(sheet.cell(row, col))
+          next if label_text.blank?
+          next unless label_matches?(label_text, normalized_labels)
+
+          return [row, col]
+        end
+      end
+
+      nil
+    end
+
+    def label_matches?(label_text, normalized_labels)
+      normalized_labels.any? do |label|
+        label_text == label ||
+          label_text.start_with?("#{label} ") ||
+          label_text.start_with?("#{label}:")
+      end
+    end
+
+    def normalize_label(value)
+      value.to_s
+           .downcase
+           .tr("\u00A0", ' ')
+           .gsub(/\s+/, ' ')
+           .gsub(/[«»]/, '')
+           .strip
+    end
+  end
+end
@@ -0,0 +1,128 @@
+module ReportParsers
+  class Form11Parser < BaseParser
+    SHEET_NAME = 'Бухгалтерский баланс'
+    FORM_CODE = '1.1'
+
+    def call
+      return {} unless sheet_exists?(SHEET_NAME)
+
+      sheet = spreadsheet.sheet(SHEET_NAME)
+      extract_year(sheet)
+      build_payloads(sheet)
+    end
+
+    private
+
+    def build_payloads(sheet)
+      header_row = find_header_row(sheet)
+      columns = detect_columns(sheet, header_row)
+
+
+      value_headers = columns[:value_columns].each_with_object({}) do |col, hash|
+        header_text = read_cell(sheet, header_row, col).to_s
+        year = extract_year_from_header(header_text)
+        hash[col] = year
+      end
+
+      all_rows = (header_row + 1).upto(sheet.last_row).filter_map do |row|
+        line_name = read_cell(sheet, row, columns[:name_column])
+        line_code = read_cell(sheet, row, columns[:code_column]).to_s.strip.presence
+
+        values = columns[:value_columns].each_with_object({}) do |col, hash|
+          value = read_cell(sheet, row, col)
+          year = value_headers[col]
+          hash[year.to_s] = value if value.present? && year.present?
+        end.compact_blank
+
+        next if line_name.blank? && line_code.blank? && values.blank?
+
+        {
+          line_name: line_name,
+          line_code: line_code,
+          values: values
+        }
+      end
+
+      payloads_by_year = {}
+      value_headers.each do |col, year|
+        next unless year.present?
+
+        rows_for_year = all_rows.map do |row|
+          value = row[:values][year.to_s]
+          next unless value.present?
+
+          {
+            line_name: row[:line_name],
+            line_code: row[:line_code],
+            value: value
+          }
+        end.compact
+
+        payloads_by_year[year] = {
+          form_code: FORM_CODE,
+          sheet_name: SHEET_NAME,
+          header_row: header_row,
+          data_year: year,
+          rows: rows_for_year
+        }
+      end
+
+      payloads_by_year
+    end
+
+    def extract_year_from_header(header_text)
+      year_match = header_text[/\b(20\d{2})\b/]
+      year_match.to_i if year_match
+    end
+
+    def find_header_row(sheet)
+      1.upto(sheet.last_row) do |row|
+        normalized_cells = 1.upto(sheet.last_column).map { |col| normalize_label(sheet.cell(row, col)) }
+        has_name = normalized_cells.any? { |text| text.include?('наименование показателя') }
+        has_code = normalized_cells.any? { |text| text == 'код' || text.start_with?('код ') || text.start_with?('код:') }
+        return row if has_name && has_code
+      end
+
+      1
+    end
+
+    def detect_columns(sheet, header_row)
+      columns = 1.upto(sheet.last_column).each_with_object({}) do |col, hash|
+        hash[col] = normalize_label(sheet.cell(header_row, col))
+      end
+
+      name_column = columns.key(columns.values.find { |text| text.include?('наименование показателя') }) || 1
+
+      code_column = columns.key(columns.values.find { |text|
+        text == 'код строки' || text.start_with?('код строки ') ||
+        (text == 'код' && columns[name_column + 1] == text)
+      }) || columns.key(columns.values.find { |text| text == 'код' || text.start_with?('код ') }) || [name_column + 1, sheet.last_column].min
+
+      value_columns = columns.keys.select do |col|
+        text = columns[col]
+        text.include?('на 31') ||
+        text.include?('на ') ||
+        text.match?(/\bна \d+ /) ||
+        text.match?(/\b20\d{2}\b/) ||
+        (col > code_column && text.include?('декабря')) ||
+        (col > code_column && text.match?(/\d+/))  # fallback: numeric values
+      end
+
+      if value_columns.empty?
+        value_columns = columns.keys.select { |col| col > code_column && columns[col].present? }
+      end
+
+      {
+        name_column: name_column,
+        code_column: code_column,
+        value_columns: value_columns.sort.first(3)
+      }
+    end
+
+    def extract_year(sheet)
+      period_text = sheet.cell(4, 1).to_s
+      year = period_text[/\d{4}/].to_i
+      yearly_file.update!(year: year) if year.positive?
+    end
+  end
+end
@@ -0,0 +1,114 @@
+module ReportParsers
+  class Form12Parser < BaseParser
+    SHEET_NAME = 'Отчет о финансовых результатах'
+    FORM_CODE = '1.2'
+
+    def call
+      return {} unless sheet_exists?(SHEET_NAME)
+
+      sheet = spreadsheet.sheet(SHEET_NAME)
+      build_payloads(sheet)
+    end
+
+    private
+
+    def build_payloads(sheet)
+      header_row = find_header_row(sheet)
+      columns = detect_columns(sheet, header_row)
+
+      value_headers = columns[:value_columns].each_with_object({}) do |col, hash|
+        header_text = read_cell(sheet, header_row, col).to_s
+        year = extract_year_from_header(header_text)
+        hash[col] = year
+      end
+
+      all_rows = (header_row + 1).upto(sheet.last_row).filter_map do |row|
+        line_name = read_cell(sheet, row, columns[:name_column])
+        line_code = read_cell(sheet, row, columns[:code_column]).to_s.strip.presence
+
+        values = columns[:value_columns].each_with_object({}) do |col, hash|
+          value = read_cell(sheet, row, col)
+          year = value_headers[col]
+          hash[year.to_s] = value if value.present? && year.present?
+        end.compact_blank
+
+        next if line_name.blank? && line_code.blank? && values.blank?
+
+        {
+          line_name: line_name,
+          line_code: line_code,
+          values: values
+        }
+      end
+
+      payloads_by_year = {}
+      value_headers.each do |col, year|
+        next unless year.present?
+
+        rows_for_year = all_rows.map do |row|
+          value = row[:values][year.to_s]
+          next unless value.present?
+
+          {
+            line_name: row[:line_name],
+            line_code: row[:line_code],
+            value: value
+          }
+        end.compact
+
+        payloads_by_year[year] = {
+          form_code: FORM_CODE,
+          sheet_name: SHEET_NAME,
+          header_row: header_row,
+          data_year: year,
+          rows: rows_for_year
+        }
+      end
+
+      payloads_by_year
+    end
+
+    def extract_year_from_header(header_text)
+      year_match = header_text[/\b(20\d{2})\b/]
+      year_match.to_i if year_match
+    end
+
+    def find_header_row(sheet)
+      1.upto(sheet.last_row) do |row|
+        normalized_cells = 1.upto(sheet.last_column).map { |col| normalize_label(sheet.cell(row, col)) }
+        has_name = normalized_cells.any? { |text| text.include?('наименование показателя') }
+        has_code = normalized_cells.any? { |text| text == 'код' || text.start_with?('код ') || text.start_with?('код:') }
+        return row if has_name && has_code
+      end
+
+      1
+    end
+
+    def detect_columns(sheet, header_row)
+      columns = 1.upto(sheet.last_column).each_with_object({}) do |col, hash|
+        hash[col] = normalize_label(sheet.cell(header_row, col))
+      end
+
+      name_column = columns.key(columns.values.find { |text| text.include?('наименование показателя') }) || 1
+
+      code_column = columns.key(columns.values.find { |text|
+        text == 'код строки' || text.start_with?('код строки ') ||
+        (text == 'код' && columns[name_column + 1] == text)
+      }) || columns.key(columns.values.find { |text| text == 'код' || text.start_with?('код ') }) || [name_column + 1, sheet.last_column].min
+
+      value_columns = columns.keys.select do |col|
+        text = columns[col]
+        next false if text.include?('поясн')
+        text.match?(/20\d{2}/) || text.include?('за ') || text.include?('на ')
+      end
+
+      value_columns = value_columns.sort.last(2)
+
+      {
+        name_column: name_column,
+        code_column: code_column,
+        value_columns: value_columns
+      }
+    end
+  end
+end
@@ -0,0 +1,112 @@
+        def extract_year_from_header(header_text)
+          year_match = header_text[/\b(20\d{2})\b/]
+          year_match.to_i if year_match
+        end
+    def find_header_row(sheet)
+      1.upto(sheet.last_row) do |row|
+        normalized_cells = 1.upto(sheet.last_column).map { |col| normalize_label(sheet.cell(row, col)) }
+        has_name = normalized_cells.any? { |text| text.include?("наименование показателя") }
+        has_code = normalized_cells.any? { |text| text == "код" || text.start_with?("код ") || text.start_with?("код:") }
+        return row if has_name && has_code
+      end
+
+      1
+    end
+
+    def detect_columns(sheet, header_row)
+      columns = 1.upto(sheet.last_column).each_with_object({}) do |col, hash|
+        hash[col] = normalize_label(sheet.cell(header_row, col))
+      end
+
+      name_column = columns.key(columns.values.find { |text| text.include?("наименование показателя") }) || 1
+
+      code_column = columns.key(columns.values.find { |text|
+        text == "код строки" || text.start_with?("код строки ") ||
+        (text == "код" && columns[name_column + 1] == text)
+      }) || columns.key(columns.values.find { |text| text == "код" || text.start_with?("код ") }) || [name_column + 1, sheet.last_column].min
+
+      value_columns = columns.keys.select do |col|
+        text = columns[col]
+        next false if text.include?("поясн")
+        text.match?(/20\d{2}/) || text.include?("за ") || text.include?("на ")
+      end
+      value_columns = value_columns.sort.last(2)
+
+      {
+        name_column: name_column,
+        code_column: code_column,
+        value_columns: value_columns
+      }
+    end
+
+    def normalize_label(text)
+      text.to_s.downcase.gsub(/[[:space:]]+/, ' ').strip
+    end
+module ReportParsers
+  class Form14Parser < BaseParser
+    SHEET_NAME = 'Отчет о движении денежных средс'
+    FORM_CODE = '1.4'
+
+    def call
+      return {} unless sheet_exists?(SHEET_NAME)
+
+      sheet = spreadsheet.sheet(SHEET_NAME)
+      build_payloads(sheet)
+    end
+
+    private
+
+    def build_payloads(sheet)
+      header_row = find_header_row(sheet)
+      columns = detect_columns(sheet, header_row)
+
+      value_headers = columns[:value_columns].each_with_object({}) do |col, hash|
+        header_text = read_cell(sheet, header_row, col).to_s
+        year = extract_year_from_header(header_text)
+        hash[col] = year
+      end
+
+      all_rows = (header_row + 1).upto(sheet.last_row).filter_map do |row|
+        line_name = read_cell(sheet, row, columns[:name_column])
+        line_code = read_cell(sheet, row, columns[:code_column]).to_s.strip.presence
+
+        values = columns[:value_columns].each_with_object({}) do |col, hash|
+          value = read_cell(sheet, row, col)
+          year = value_headers[col]
+          hash[year.to_s] = value if value.present? && year.present?
+        end.compact_blank
+
+        next if line_name.blank? && line_code.blank? && values.blank?
+
+        {
+          line_name: line_name,
+          line_code: line_code,
+          values: values
+        }
+      end
+
+      payloads_by_year = {}
+      value_headers.each do |col, year|
+        next unless year.present?
+
+        rows_for_year = all_rows.map do |row|
+          value = row[:values][year.to_s]
+          next unless value.present?
+
+          {
+            line_name: row[:line_name],
+            line_code: row[:line_code],
+            value: value
+          }
+        end.compact
+
+        payloads_by_year[year] = {
+          form_code: FORM_CODE,
+          rows: rows_for_year
+        }
+      end
+
+      payloads_by_year
+    end
+  end
+end
@@ -0,0 +1,73 @@
+module ReportParsers
+  class OrganizationInfoParser < BaseParser
+    SHEET_NAME = 'Сведения об организации'
+
+    ORGANIZATION_LABELS = {
+      full_name: ['Полное наименование юридического лица'],
+      inn:       ['ИНН'],
+      kpp:       ['КПП'],
+      okpo:      ['Код по ОКПО'],
+      okfs:      ['Форма собственности (по ОКФС)'],
+      okopf:     ['Организационно-правовая форма (по ОКОПФ)'],
+      okved2:    ['Вид экономической деятельности по ОКВЭД 2'],
+      address:   ['Местонахождение (адрес)']
+    }.freeze
+
+    AUDITOR_LABELS = {
+      name: ['Наименование аудиторской организации/ФИО индивидуального аудитора'],
+      inn:  ['ИНН'],
+      ogrn: ['ОГРН/ОГРНИП']
+    }.freeze
+
+    def call
+      return unless sheet_exists?(SHEET_NAME)
+
+      sheet = spreadsheet.sheet(SHEET_NAME)
+      parse_organization(sheet)
+      parse_auditor(sheet)
+    end
+
+    private
+
+    def parse_organization(sheet)
+      end_row = auditor_section_start_row(sheet) ? auditor_section_start_row(sheet) - 1 : sheet.last_row
+      org_data = ORGANIZATION_LABELS.each_with_object({}) do |(attr, labels), hash|
+        hash[attr] = find_value_by_labels(sheet, labels, row_range: 1..end_row)
+      end
+
+      yearly_file.organization.update!(org_data)
+    end
+
+    def parse_auditor(sheet)
+      start_row = auditor_section_start_row(sheet) || 1
+      auditor_data = AUDITOR_LABELS.each_with_object({}) do |(attr, labels), hash|
+        hash[attr] = find_value_by_labels(sheet, labels, row_range: start_row..sheet.last_row)
+      end
+
+      return if auditor_data[:name].blank?
+
+      auditor_inn = auditor_data[:inn].to_s.strip
+      auditor = if auditor_inn.present?
+                  Auditor.find_or_initialize_by(inn: auditor_inn)
+                else
+                  Auditor.find_or_initialize_by(name: auditor_data[:name])
+                end
+
+      auditor.assign_attributes(
+        name: auditor_data[:name],
+        inn: auditor_inn.presence,
+        ogrn: auditor_data[:ogrn].to_s.strip.presence
+      )
+      auditor.save!
+
+      yearly_file.update!(auditor: auditor)
+    end
+
+    def auditor_section_start_row(sheet)
+      @auditor_section_start_row ||= begin
+        labels = AUDITOR_LABELS.fetch(:name)
+        find_label_position(sheet, labels, row_range: 1..sheet.last_row)&.first
+      end
+    end
+  end
+end
@@ -0,0 +1,9 @@
+class AddUniqueIndexToFinancialStatements < ActiveRecord::Migration[7.1]
+  def change
+    add_index :financial_statements,
+      [:yearly_file_id, :document_type_id, :sheet_number],
+      name: 'index_fs_on_yearly_file_doc_type_sheet',
+      unique: true,
+      if_not_exists: true
+  end
+end
@@ -0,0 +1,18 @@
+class SeedDocumentTypes < ActiveRecord::Migration[7.0]
+  def up
+    execute <<-SQL
+      INSERT INTO document_types (id, name, created_at, updated_at) VALUES
+        (1, 'Бухгалтерский баланс', NOW(), NOW()),
+        (2, 'Отчет о финансовых результатах', NOW(), NOW()),
+        (3, 'Отчет об изменениях капитала', NOW(), NOW()),
+        (4, 'Отчет о движении денежных средств', NOW(), NOW())
+      ON CONFLICT (id) DO NOTHING;
+    SQL
+  end
+
+  def down
+    execute <<-SQL
+      DELETE FROM document_types WHERE id IN (1,2,3,4);
+    SQL
+  end
+end
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.

-ActiveRecord::Schema[7.1].define(version: 2026_03_12_000005) do
+ActiveRecord::Schema[7.1].define(version: 2026_03_25_063000) do
  # These are extensions that must be enabled in order to support this database
  enable_extension "plpgsql"

@@ -65,6 +65,7 @@ ActiveRecord::Schema[7.1].define(version: 2026_03_12_000005) do
    t.datetime "created_at", null: false
    t.datetime "updated_at", null: false
    t.index ["document_type_id"], name: "index_financial_statements_on_document_type_id"
+    t.index ["yearly_file_id", "document_type_id", "sheet_number"], name: "index_fs_on_yearly_file_doc_type_sheet", unique: true
    t.index ["yearly_file_id"], name: "index_financial_statements_on_yearly_file_id"
  end