SU2-19 | Парсинг двух форм, пока без понимания того, что делать с расхожениями в данных (обе формы в одном файле)

app/services/report_parser.rb

@@ -1,33 +1,18 @@
 require 'roo'
 
 class ReportParser
-  SHEET_NAME = 'Сведения об организации'
-  CELL_MAP = {
-    full_name: { row: 6,  col: 13 },
-    inn:       { row: 10, col: 13 },
-    kpp:       { row: 11, col: 13 },
-    okpo:      { row: 12, col: 13 },
-    okfs:      { row: 13, col: 13 },
-    okopf:     { row: 14, col: 13 },
-    okved2:    { row: 15, col: 13 },
-    address:   { row: 16, col: 13 }
-  }.freeze
-
-  AUDITOR_MAP = {
-    name: { row: 19, col: 13 },
-    inn:  { row: 20, col: 13 },
-    ogrn: { row: 21, col: 13 }
-  }.freeze
-
   def initialize(yearly_file)
     @yearly_file = yearly_file
   end
 
   def call
     ActiveRecord::Base.transaction do
-      parse_organization
+      ReportParsers::OrganizationInfoParser.new(@yearly_file, spreadsheet: spreadsheet).call
-      parse_auditor
+
-      extract_year
+      form_11_data = ReportParsers::Form11Parser.new(@yearly_file, spreadsheet: spreadsheet).call
+      form_12_data = ReportParsers::Form12Parser.new(@yearly_file, spreadsheet: spreadsheet).call
+
+      merge_financial_statements(form_11_data, form_12_data)
     end
   end
 
@@ -43,56 +28,48 @@ class ReportParser
     end
   end
 
-  def info_sheet
+  def merge_financial_statements(form_11_data, form_12_data)
-    @info_sheet ||= spreadsheet.sheet(SHEET_NAME)
+    document_type = DocumentType.find_or_create_by!(name: 'Финансовые данные')
+
+    all_years = (form_11_data.keys + form_12_data.keys).uniq
+
+    all_years.each do |year|
+      statement = find_or_build_statement(document_type, year)
+
+      all_rows = []
+
+      if form_11_data[year].present?
+        form_11_rows = form_11_data[year][:rows] || form_11_data[year]['rows'] || []
+        all_rows.concat(form_11_rows)
       end
 
-  def read_cell(sheet, row, col)
+      if form_12_data[year].present?
-    value = sheet.cell(row, col)
+        form_12_rows = form_12_data[year][:rows] || form_12_data[year]['rows'] || []
-    value.is_a?(String) ? value.strip.presence : value
+        all_rows.concat(form_12_rows)
       end
 
-  def parse_organization
+      if all_rows.present?
-    org_data = CELL_MAP.each_with_object({}) do |(attr, pos), hash|
+        payload = {
-      hash[attr] = read_cell(info_sheet, pos[:row], pos[:col])
+          data_year: year,
-    end
+          rows: all_rows
+        }
 
-    organization = @yearly_file.organization
+        statement.assign_attributes(data: payload)
-    organization.update!(org_data)
+        statement.save!
-    organization
+      end
-  end
+    end
-
+  end
-  def parse_auditor
+
-    auditor_data = AUDITOR_MAP.each_with_object({}) do |(attr, pos), hash|
+  def find_or_build_statement(document_type, year)
-      hash[attr] = read_cell(info_sheet, pos[:row], pos[:col])
+    existing = FinancialStatement
-    end
+               .joins(:yearly_file)
-
+               .where(document_type: document_type, sheet_number: year)
-    return if auditor_data[:name].blank?
+               .where(yearly_files: { organization_id: @yearly_file.organization_id })
-
+               .order('yearly_files.year DESC NULLS LAST, financial_statements.updated_at DESC')
-    auditor_inn = auditor_data[:inn].to_s.strip
+               .first
-    auditor = if auditor_inn.present?
+
-                Auditor.find_or_initialize_by(inn: auditor_inn)
+    return existing if existing
-              else
+
-                Auditor.find_or_initialize_by(name: auditor_data[:name])
+    @yearly_file.financial_statements.new(document_type: document_type, sheet_number: year)
-              end
-
-    auditor.assign_attributes(
-      name: auditor_data[:name],
-      inn: auditor_inn.presence,
-      ogrn: auditor_data[:ogrn].to_s.strip.presence
-    )
-    auditor.save!
-
-    @yearly_file.update!(auditor: auditor)
-    auditor
-  end
-
-  def extract_year
-    balance_sheet = spreadsheet.sheet('Бухгалтерский баланс')
-    period_text = balance_sheet.cell(4, 1).to_s
-    year = period_text.match(/(\d{4})/)&.captures&.first&.to_i
-
-    @yearly_file.update!(year: year) if year
-    year
   end
 end

app/services/report_parsers/base_parser.rb

@@ -0,0 +1,100 @@
+require 'set'
+
+module ReportParsers
+  class BaseParser
+    def initialize(yearly_file, spreadsheet:)
+      @yearly_file = yearly_file
+      @spreadsheet = spreadsheet
+    end
+
+    private
+
+    attr_reader :yearly_file, :spreadsheet
+
+    def sheet_exists?(sheet_name)
+      spreadsheet.sheets.include?(sheet_name)
+    end
+
+    def read_cell(sheet, row, col)
+      value = sheet.cell(row, col)
+
+      # Handle merged cells: if value is nil/blank, try reading from nearby cells
+      if value.blank? && sheet.respond_to?(:merged_cells)
+        value = find_merged_cell_value(sheet, row, col)
+      end
+
+      value.is_a?(String) ? value.strip.presence : value
+    end
+
+    def find_merged_cell_value(sheet, row, col)
+      # Try to find value in merged cell range
+      # Check cells in a small radius around the target cell
+      (-1..1).each do |row_offset|
+        (-1..1).each do |col_offset|
+          next if row_offset == 0 && col_offset == 0
+
+          check_row = row + row_offset
+          check_col = col + col_offset
+
+          next if check_row < 1 || check_col < 1
+          next if check_row > sheet.last_row || check_col > sheet.last_column
+
+          value = sheet.cell(check_row, check_col)
+          return value if value.present?
+        end
+      end
+
+      nil
+    end
+
+    def find_value_by_labels(sheet, labels, row_range:, value_offset: 1)
+      position = find_label_position(sheet, labels, row_range: row_range)
+      return if position.nil?
+
+      row, col = position
+
+      max_col = sheet.last_column.to_i
+      (col + value_offset).upto(max_col) do |search_col|
+        value = read_cell(sheet, row, search_col)
+        return value if value.present?
+      end
+
+      nil
+    end
+
+    def find_label_position(sheet, labels, row_range:)
+      normalized_labels = Array(labels).map { |label| normalize_label(label) }.to_set
+      max_col = sheet.last_column.to_i
+      return if max_col <= 1
+
+      row_range.each do |row|
+        1.upto(max_col - 1) do |col|
+          label_text = normalize_label(sheet.cell(row, col))
+          next if label_text.blank?
+          next unless label_matches?(label_text, normalized_labels)
+
+          return [row, col]
+        end
+      end
+
+      nil
+    end
+
+    def label_matches?(label_text, normalized_labels)
+      normalized_labels.any? do |label|
+        label_text == label ||
+          label_text.start_with?("#{label} ") ||
+          label_text.start_with?("#{label}:")
+      end
+    end
+
+    def normalize_label(value)
+      value.to_s
+           .downcase
+           .tr("\u00A0", ' ')
+           .gsub(/\s+/, ' ')
+           .gsub(/[«»]/, '')
+           .strip
+    end
+  end
+end

app/services/report_parsers/form11_parser.rb

@@ -0,0 +1,128 @@
+module ReportParsers
+  class Form11Parser < BaseParser
+    SHEET_NAME = 'Бухгалтерский баланс'
+    FORM_CODE = '1.1'
+
+    def call
+      return {} unless sheet_exists?(SHEET_NAME)
+
+      sheet = spreadsheet.sheet(SHEET_NAME)
+      extract_year(sheet)
+      build_payloads(sheet)
+    end
+
+    private
+
+    def build_payloads(sheet)
+      header_row = find_header_row(sheet)
+      columns = detect_columns(sheet, header_row)
+
+
+      value_headers = columns[:value_columns].each_with_object({}) do |col, hash|
+        header_text = read_cell(sheet, header_row, col).to_s
+        year = extract_year_from_header(header_text)
+        hash[col] = year
+      end
+
+      all_rows = (header_row + 1).upto(sheet.last_row).filter_map do |row|
+        line_name = read_cell(sheet, row, columns[:name_column])
+        line_code = read_cell(sheet, row, columns[:code_column]).to_s.strip.presence
+
+        values = columns[:value_columns].each_with_object({}) do |col, hash|
+          value = read_cell(sheet, row, col)
+          year = value_headers[col]
+          hash[year.to_s] = value if value.present? && year.present?
+        end.compact_blank
+
+        next if line_name.blank? && line_code.blank? && values.blank?
+
+        {
+          line_name: line_name,
+          line_code: line_code,
+          values: values
+        }
+      end
+
+      payloads_by_year = {}
+      value_headers.each do |col, year|
+        next unless year.present?
+
+        rows_for_year = all_rows.map do |row|
+          value = row[:values][year.to_s]
+          next unless value.present?
+
+          {
+            line_name: row[:line_name],
+            line_code: row[:line_code],
+            value: value
+          }
+        end.compact
+
+        payloads_by_year[year] = {
+          form_code: FORM_CODE,
+          sheet_name: SHEET_NAME,
+          header_row: header_row,
+          data_year: year,
+          rows: rows_for_year
+        }
+      end
+
+      payloads_by_year
+    end
+
+    def extract_year_from_header(header_text)
+      year_match = header_text[/\b(20\d{2})\b/]
+      year_match.to_i if year_match
+    end
+
+    def find_header_row(sheet)
+      1.upto(sheet.last_row) do |row|
+        normalized_cells = 1.upto(sheet.last_column).map { |col| normalize_label(sheet.cell(row, col)) }
+        has_name = normalized_cells.any? { |text| text.include?('наименование показателя') }
+        has_code = normalized_cells.any? { |text| text == 'код' || text.start_with?('код ') || text.start_with?('код:') }
+        return row if has_name && has_code
+      end
+
+      1
+    end
+
+    def detect_columns(sheet, header_row)
+      columns = 1.upto(sheet.last_column).each_with_object({}) do |col, hash|
+        hash[col] = normalize_label(sheet.cell(header_row, col))
+      end
+
+      name_column = columns.key(columns.values.find { |text| text.include?('наименование показателя') }) || 1
+
+      code_column = columns.key(columns.values.find { |text|
+        text == 'код строки' || text.start_with?('код строки ') ||
+        (text == 'код' && columns[name_column + 1] == text)
+      }) || columns.key(columns.values.find { |text| text == 'код' || text.start_with?('код ') }) || [name_column + 1, sheet.last_column].min
+
+      value_columns = columns.keys.select do |col|
+        text = columns[col]
+        text.include?('на 31') ||
+        text.include?('на ') ||
+        text.match?(/\bна \d+ /) ||
+        text.match?(/\b20\d{2}\b/) ||
+        (col > code_column && text.include?('декабря')) ||
+        (col > code_column && text.match?(/\d+/))  # fallback: numeric values
+      end
+
+      if value_columns.empty?
+        value_columns = columns.keys.select { |col| col > code_column && columns[col].present? }
+      end
+
+      {
+        name_column: name_column,
+        code_column: code_column,
+        value_columns: value_columns.sort.first(3)
+      }
+    end
+
+    def extract_year(sheet)
+      period_text = sheet.cell(4, 1).to_s
+      year = period_text[/\d{4}/].to_i
+      yearly_file.update!(year: year) if year.positive?
+    end
+  end
+end

app/services/report_parsers/form12_parser.rb

@@ -0,0 +1,119 @@
+module ReportParsers
+  class Form12Parser < BaseParser
+    SHEET_NAME = 'Отчет о финансовых результатах'
+    FORM_CODE = '1.2'
+
+    def call
+      return {} unless sheet_exists?(SHEET_NAME)
+
+      sheet = spreadsheet.sheet(SHEET_NAME)
+      build_payloads(sheet)
+    end
+
+    private
+
+    def build_payloads(sheet)
+      header_row = find_header_row(sheet)
+      columns = detect_columns(sheet, header_row)
+
+      value_headers = columns[:value_columns].each_with_object({}) do |col, hash|
+        header_text = read_cell(sheet, header_row, col).to_s
+        year = extract_year_from_header(header_text)
+        hash[col] = year
+      end
+
+      all_rows = (header_row + 1).upto(sheet.last_row).filter_map do |row|
+        line_name = read_cell(sheet, row, columns[:name_column])
+        line_code = read_cell(sheet, row, columns[:code_column]).to_s.strip.presence
+
+        values = columns[:value_columns].each_with_object({}) do |col, hash|
+          value = read_cell(sheet, row, col)
+          year = value_headers[col]
+          hash[year.to_s] = value if value.present? && year.present?
+        end.compact_blank
+
+        next if line_name.blank? && line_code.blank? && values.blank?
+
+        {
+          line_name: line_name,
+          line_code: line_code,
+          values: values
+        }
+      end
+
+      payloads_by_year = {}
+      value_headers.each do |col, year|
+        next unless year.present?
+
+        rows_for_year = all_rows.map do |row|
+          value = row[:values][year.to_s]
+          next unless value.present?
+
+          {
+            line_name: row[:line_name],
+            line_code: row[:line_code],
+            value: value
+          }
+        end.compact
+
+        payloads_by_year[year] = {
+          form_code: FORM_CODE,
+          sheet_name: SHEET_NAME,
+          header_row: header_row,
+          data_year: year,
+          rows: rows_for_year
+        }
+      end
+
+      payloads_by_year
+    end
+
+    def extract_year_from_header(header_text)
+      year_match = header_text[/\b(20\d{2})\b/]
+      year_match.to_i if year_match
+    end
+
+    def find_header_row(sheet)
+      1.upto(sheet.last_row) do |row|
+        normalized_cells = 1.upto(sheet.last_column).map { |col| normalize_label(sheet.cell(row, col)) }
+        has_name = normalized_cells.any? { |text| text.include?('наименование показателя') }
+        has_code = normalized_cells.any? { |text| text == 'код' || text.start_with?('код ') || text.start_with?('код:') }
+        return row if has_name && has_code
+      end
+
+      1
+    end
+
+    def detect_columns(sheet, header_row)
+      columns = 1.upto(sheet.last_column).each_with_object({}) do |col, hash|
+        hash[col] = normalize_label(sheet.cell(header_row, col))
+      end
+
+      name_column = columns.key(columns.values.find { |text| text.include?('наименование показателя') }) || 1
+
+      code_column = columns.key(columns.values.find { |text|
+        text == 'код строки' || text.start_with?('код строки ') ||
+        (text == 'код' && columns[name_column + 1] == text)
+      }) || columns.key(columns.values.find { |text| text == 'код' || text.start_with?('код ') }) || [name_column + 1, sheet.last_column].min
+
+      value_columns = columns.keys.select do |col|
+        text = columns[col]
+        text.include?('за ') ||
+        text.include?('на ') ||
+        text.match?(/\b20\d{2}\b/) ||
+        (col > code_column && text.include?('года')) ||
+        (col > code_column && text.match?(/\d+/))
+      end
+
+      if value_columns.empty?
+        value_columns = columns.keys.select { |col| col > code_column && columns[col].present? }
+      end
+
+      {
+        name_column: name_column,
+        code_column: code_column,
+        value_columns: value_columns.sort.first(3)
+      }
+    end
+  end
+end

app/services/report_parsers/organization_info_parser.rb

@@ -0,0 +1,73 @@
+module ReportParsers
+  class OrganizationInfoParser < BaseParser
+    SHEET_NAME = 'Сведения об организации'
+
+    ORGANIZATION_LABELS = {
+      full_name: ['Полное наименование юридического лица'],
+      inn:       ['ИНН'],
+      kpp:       ['КПП'],
+      okpo:      ['Код по ОКПО'],
+      okfs:      ['Форма собственности (по ОКФС)'],
+      okopf:     ['Организационно-правовая форма (по ОКОПФ)'],
+      okved2:    ['Вид экономической деятельности по ОКВЭД 2'],
+      address:   ['Местонахождение (адрес)']
+    }.freeze
+
+    AUDITOR_LABELS = {
+      name: ['Наименование аудиторской организации/ФИО индивидуального аудитора'],
+      inn:  ['ИНН'],
+      ogrn: ['ОГРН/ОГРНИП']
+    }.freeze
+
+    def call
+      return unless sheet_exists?(SHEET_NAME)
+
+      sheet = spreadsheet.sheet(SHEET_NAME)
+      parse_organization(sheet)
+      parse_auditor(sheet)
+    end
+
+    private
+
+    def parse_organization(sheet)
+      end_row = auditor_section_start_row(sheet) ? auditor_section_start_row(sheet) - 1 : sheet.last_row
+      org_data = ORGANIZATION_LABELS.each_with_object({}) do |(attr, labels), hash|
+        hash[attr] = find_value_by_labels(sheet, labels, row_range: 1..end_row)
+      end
+
+      yearly_file.organization.update!(org_data)
+    end
+
+    def parse_auditor(sheet)
+      start_row = auditor_section_start_row(sheet) || 1
+      auditor_data = AUDITOR_LABELS.each_with_object({}) do |(attr, labels), hash|
+        hash[attr] = find_value_by_labels(sheet, labels, row_range: start_row..sheet.last_row)
+      end
+
+      return if auditor_data[:name].blank?
+
+      auditor_inn = auditor_data[:inn].to_s.strip
+      auditor = if auditor_inn.present?
+                  Auditor.find_or_initialize_by(inn: auditor_inn)
+                else
+                  Auditor.find_or_initialize_by(name: auditor_data[:name])
+                end
+
+      auditor.assign_attributes(
+        name: auditor_data[:name],
+        inn: auditor_inn.presence,
+        ogrn: auditor_data[:ogrn].to_s.strip.presence
+      )
+      auditor.save!
+
+      yearly_file.update!(auditor: auditor)
+    end
+
+    def auditor_section_start_row(sheet)
+      @auditor_section_start_row ||= begin
+        labels = AUDITOR_LABELS.fetch(:name)
+        find_label_position(sheet, labels, row_range: 1..sheet.last_row)&.first
+      end
+    end
+  end
+end

db/migrate/20260321000006_add_unique_index_to_financial_statements.rb

@@ -0,0 +1,9 @@
+class AddUniqueIndexToFinancialStatements < ActiveRecord::Migration[7.1]
+  def change
+    add_index :financial_statements,
+      [:yearly_file_id, :document_type_id, :sheet_number],
+      name: 'index_fs_on_yearly_file_doc_type_sheet',
+      unique: true,
+      if_not_exists: true
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.1].define(version: 2026_03_12_000005) do
+ActiveRecord::Schema[7.1].define(version: 2026_03_21_000006) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
@@ -65,6 +65,7 @@ ActiveRecord::Schema[7.1].define(version: 2026_03_12_000005) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["document_type_id"], name: "index_financial_statements_on_document_type_id"
+    t.index ["yearly_file_id", "document_type_id", "sheet_number"], name: "index_fs_on_yearly_file_doc_type_sheet", unique: true
     t.index ["yearly_file_id"], name: "index_financial_statements_on_yearly_file_id"
   end