From 207e4b45a7e5e950b213ea1b806997902458aa1f Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sat, 21 Mar 2026 15:26:37 +0300 Subject: [PATCH] =?UTF-8?q?SU2-19=20|=20=D0=9F=D0=B0=D1=80=D1=81=D0=B8?= =?UTF-8?q?=D0=BD=D0=B3=20=D0=B4=D0=B2=D1=83=D1=85=20=D1=84=D0=BE=D1=80?= =?UTF-8?q?=D0=BC,=20=D0=BF=D0=BE=D0=BA=D0=B0=20=D0=B1=D0=B5=D0=B7=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=BD=D0=B8=D0=BC=D0=B0=D0=BD=D0=B8=D1=8F=20=D1=82?= =?UTF-8?q?=D0=BE=D0=B3=D0=BE,=20=D1=87=D1=82=D0=BE=20=D0=B4=D0=B5=D0=BB?= =?UTF-8?q?=D0=B0=D1=82=D1=8C=20=D1=81=20=D1=80=D0=B0=D1=81=D1=85=D0=BE?= =?UTF-8?q?=D0=B6=D0=B5=D0=BD=D0=B8=D1=8F=D0=BC=D0=B8=20=D0=B2=20=D0=B4?= =?UTF-8?q?=D0=B0=D0=BD=D0=BD=D1=8B=D1=85=20(=D0=BE=D0=B1=D0=B5=20=D1=84?= =?UTF-8?q?=D0=BE=D1=80=D0=BC=D1=8B=20=D0=B2=20=D0=BE=D0=B4=D0=BD=D0=BE?= =?UTF-8?q?=D0=BC=20=D1=84=D0=B0=D0=B9=D0=BB=D0=B5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/report_parser.rb | 107 ++++++--------- app/services/report_parsers/base_parser.rb | 100 ++++++++++++++ app/services/report_parsers/form11_parser.rb | 128 ++++++++++++++++++ app/services/report_parsers/form12_parser.rb | 119 ++++++++++++++++ .../organization_info_parser.rb | 73 ++++++++++ ...dd_unique_index_to_financial_statements.rb | 9 ++ db/schema.rb | 3 +- 7 files changed, 473 insertions(+), 66 deletions(-) create mode 100644 app/services/report_parsers/base_parser.rb create mode 100644 app/services/report_parsers/form11_parser.rb create mode 100644 app/services/report_parsers/form12_parser.rb create mode 100644 app/services/report_parsers/organization_info_parser.rb create mode 100644 db/migrate/20260321000006_add_unique_index_to_financial_statements.rb diff --git a/app/services/report_parser.rb b/app/services/report_parser.rb index f277c92..c1dc75e 100644 --- a/app/services/report_parser.rb +++ b/app/services/report_parser.rb @@ -1,33 +1,18 @@ require 'roo' class ReportParser - SHEET_NAME = 'Сведения об организации' - CELL_MAP = { - full_name: { row: 6, col: 13 }, - inn: { row: 10, col: 13 }, - kpp: { row: 11, col: 13 }, - okpo: { row: 12, col: 13 }, - okfs: { row: 13, col: 13 }, - okopf: { row: 14, col: 13 }, - okved2: { row: 15, col: 13 }, - address: { row: 16, col: 13 } - }.freeze - - AUDITOR_MAP = { - name: { row: 19, col: 13 }, - inn: { row: 20, col: 13 }, - ogrn: { row: 21, col: 13 } - }.freeze - def initialize(yearly_file) @yearly_file = yearly_file end def call ActiveRecord::Base.transaction do - parse_organization - parse_auditor - extract_year + ReportParsers::OrganizationInfoParser.new(@yearly_file, spreadsheet: spreadsheet).call + + form_11_data = ReportParsers::Form11Parser.new(@yearly_file, spreadsheet: spreadsheet).call + form_12_data = ReportParsers::Form12Parser.new(@yearly_file, spreadsheet: spreadsheet).call + + merge_financial_statements(form_11_data, form_12_data) end end @@ -43,56 +28,48 @@ class ReportParser end end - def info_sheet - @info_sheet ||= spreadsheet.sheet(SHEET_NAME) - end + def merge_financial_statements(form_11_data, form_12_data) + document_type = DocumentType.find_or_create_by!(name: 'Финансовые данные') - def read_cell(sheet, row, col) - value = sheet.cell(row, col) - value.is_a?(String) ? value.strip.presence : value - end + all_years = (form_11_data.keys + form_12_data.keys).uniq - def parse_organization - org_data = CELL_MAP.each_with_object({}) do |(attr, pos), hash| - hash[attr] = read_cell(info_sheet, pos[:row], pos[:col]) + all_years.each do |year| + statement = find_or_build_statement(document_type, year) + + all_rows = [] + + if form_11_data[year].present? + form_11_rows = form_11_data[year][:rows] || form_11_data[year]['rows'] || [] + all_rows.concat(form_11_rows) + end + + if form_12_data[year].present? + form_12_rows = form_12_data[year][:rows] || form_12_data[year]['rows'] || [] + all_rows.concat(form_12_rows) + end + + if all_rows.present? + payload = { + data_year: year, + rows: all_rows + } + + statement.assign_attributes(data: payload) + statement.save! + end end - - organization = @yearly_file.organization - organization.update!(org_data) - organization end - def parse_auditor - auditor_data = AUDITOR_MAP.each_with_object({}) do |(attr, pos), hash| - hash[attr] = read_cell(info_sheet, pos[:row], pos[:col]) - end + def find_or_build_statement(document_type, year) + existing = FinancialStatement + .joins(:yearly_file) + .where(document_type: document_type, sheet_number: year) + .where(yearly_files: { organization_id: @yearly_file.organization_id }) + .order('yearly_files.year DESC NULLS LAST, financial_statements.updated_at DESC') + .first - return if auditor_data[:name].blank? + return existing if existing - auditor_inn = auditor_data[:inn].to_s.strip - auditor = if auditor_inn.present? - Auditor.find_or_initialize_by(inn: auditor_inn) - else - Auditor.find_or_initialize_by(name: auditor_data[:name]) - end - - auditor.assign_attributes( - name: auditor_data[:name], - inn: auditor_inn.presence, - ogrn: auditor_data[:ogrn].to_s.strip.presence - ) - auditor.save! - - @yearly_file.update!(auditor: auditor) - auditor - end - - def extract_year - balance_sheet = spreadsheet.sheet('Бухгалтерский баланс') - period_text = balance_sheet.cell(4, 1).to_s - year = period_text.match(/(\d{4})/)&.captures&.first&.to_i - - @yearly_file.update!(year: year) if year - year + @yearly_file.financial_statements.new(document_type: document_type, sheet_number: year) end end diff --git a/app/services/report_parsers/base_parser.rb b/app/services/report_parsers/base_parser.rb new file mode 100644 index 0000000..232bc1b --- /dev/null +++ b/app/services/report_parsers/base_parser.rb @@ -0,0 +1,100 @@ +require 'set' + +module ReportParsers + class BaseParser + def initialize(yearly_file, spreadsheet:) + @yearly_file = yearly_file + @spreadsheet = spreadsheet + end + + private + + attr_reader :yearly_file, :spreadsheet + + def sheet_exists?(sheet_name) + spreadsheet.sheets.include?(sheet_name) + end + + def read_cell(sheet, row, col) + value = sheet.cell(row, col) + + # Handle merged cells: if value is nil/blank, try reading from nearby cells + if value.blank? && sheet.respond_to?(:merged_cells) + value = find_merged_cell_value(sheet, row, col) + end + + value.is_a?(String) ? value.strip.presence : value + end + + def find_merged_cell_value(sheet, row, col) + # Try to find value in merged cell range + # Check cells in a small radius around the target cell + (-1..1).each do |row_offset| + (-1..1).each do |col_offset| + next if row_offset == 0 && col_offset == 0 + + check_row = row + row_offset + check_col = col + col_offset + + next if check_row < 1 || check_col < 1 + next if check_row > sheet.last_row || check_col > sheet.last_column + + value = sheet.cell(check_row, check_col) + return value if value.present? + end + end + + nil + end + + def find_value_by_labels(sheet, labels, row_range:, value_offset: 1) + position = find_label_position(sheet, labels, row_range: row_range) + return if position.nil? + + row, col = position + + max_col = sheet.last_column.to_i + (col + value_offset).upto(max_col) do |search_col| + value = read_cell(sheet, row, search_col) + return value if value.present? + end + + nil + end + + def find_label_position(sheet, labels, row_range:) + normalized_labels = Array(labels).map { |label| normalize_label(label) }.to_set + max_col = sheet.last_column.to_i + return if max_col <= 1 + + row_range.each do |row| + 1.upto(max_col - 1) do |col| + label_text = normalize_label(sheet.cell(row, col)) + next if label_text.blank? + next unless label_matches?(label_text, normalized_labels) + + return [row, col] + end + end + + nil + end + + def label_matches?(label_text, normalized_labels) + normalized_labels.any? do |label| + label_text == label || + label_text.start_with?("#{label} ") || + label_text.start_with?("#{label}:") + end + end + + def normalize_label(value) + value.to_s + .downcase + .tr("\u00A0", ' ') + .gsub(/\s+/, ' ') + .gsub(/[«»]/, '') + .strip + end + end +end diff --git a/app/services/report_parsers/form11_parser.rb b/app/services/report_parsers/form11_parser.rb new file mode 100644 index 0000000..0185894 --- /dev/null +++ b/app/services/report_parsers/form11_parser.rb @@ -0,0 +1,128 @@ +module ReportParsers + class Form11Parser < BaseParser + SHEET_NAME = 'Бухгалтерский баланс' + FORM_CODE = '1.1' + + def call + return {} unless sheet_exists?(SHEET_NAME) + + sheet = spreadsheet.sheet(SHEET_NAME) + extract_year(sheet) + build_payloads(sheet) + end + + private + + def build_payloads(sheet) + header_row = find_header_row(sheet) + columns = detect_columns(sheet, header_row) + + + value_headers = columns[:value_columns].each_with_object({}) do |col, hash| + header_text = read_cell(sheet, header_row, col).to_s + year = extract_year_from_header(header_text) + hash[col] = year + end + + all_rows = (header_row + 1).upto(sheet.last_row).filter_map do |row| + line_name = read_cell(sheet, row, columns[:name_column]) + line_code = read_cell(sheet, row, columns[:code_column]).to_s.strip.presence + + values = columns[:value_columns].each_with_object({}) do |col, hash| + value = read_cell(sheet, row, col) + year = value_headers[col] + hash[year.to_s] = value if value.present? && year.present? + end.compact_blank + + next if line_name.blank? && line_code.blank? && values.blank? + + { + line_name: line_name, + line_code: line_code, + values: values + } + end + + payloads_by_year = {} + value_headers.each do |col, year| + next unless year.present? + + rows_for_year = all_rows.map do |row| + value = row[:values][year.to_s] + next unless value.present? + + { + line_name: row[:line_name], + line_code: row[:line_code], + value: value + } + end.compact + + payloads_by_year[year] = { + form_code: FORM_CODE, + sheet_name: SHEET_NAME, + header_row: header_row, + data_year: year, + rows: rows_for_year + } + end + + payloads_by_year + end + + def extract_year_from_header(header_text) + year_match = header_text[/\b(20\d{2})\b/] + year_match.to_i if year_match + end + + def find_header_row(sheet) + 1.upto(sheet.last_row) do |row| + normalized_cells = 1.upto(sheet.last_column).map { |col| normalize_label(sheet.cell(row, col)) } + has_name = normalized_cells.any? { |text| text.include?('наименование показателя') } + has_code = normalized_cells.any? { |text| text == 'код' || text.start_with?('код ') || text.start_with?('код:') } + return row if has_name && has_code + end + + 1 + end + + def detect_columns(sheet, header_row) + columns = 1.upto(sheet.last_column).each_with_object({}) do |col, hash| + hash[col] = normalize_label(sheet.cell(header_row, col)) + end + + name_column = columns.key(columns.values.find { |text| text.include?('наименование показателя') }) || 1 + + code_column = columns.key(columns.values.find { |text| + text == 'код строки' || text.start_with?('код строки ') || + (text == 'код' && columns[name_column + 1] == text) + }) || columns.key(columns.values.find { |text| text == 'код' || text.start_with?('код ') }) || [name_column + 1, sheet.last_column].min + + value_columns = columns.keys.select do |col| + text = columns[col] + text.include?('на 31') || + text.include?('на ') || + text.match?(/\bна \d+ /) || + text.match?(/\b20\d{2}\b/) || + (col > code_column && text.include?('декабря')) || + (col > code_column && text.match?(/\d+/)) # fallback: numeric values + end + + if value_columns.empty? + value_columns = columns.keys.select { |col| col > code_column && columns[col].present? } + end + + { + name_column: name_column, + code_column: code_column, + value_columns: value_columns.sort.first(3) + } + end + + def extract_year(sheet) + period_text = sheet.cell(4, 1).to_s + year = period_text[/\d{4}/].to_i + yearly_file.update!(year: year) if year.positive? + end + end +end diff --git a/app/services/report_parsers/form12_parser.rb b/app/services/report_parsers/form12_parser.rb new file mode 100644 index 0000000..eda69e0 --- /dev/null +++ b/app/services/report_parsers/form12_parser.rb @@ -0,0 +1,119 @@ +module ReportParsers + class Form12Parser < BaseParser + SHEET_NAME = 'Отчет о финансовых результатах' + FORM_CODE = '1.2' + + def call + return {} unless sheet_exists?(SHEET_NAME) + + sheet = spreadsheet.sheet(SHEET_NAME) + build_payloads(sheet) + end + + private + + def build_payloads(sheet) + header_row = find_header_row(sheet) + columns = detect_columns(sheet, header_row) + + value_headers = columns[:value_columns].each_with_object({}) do |col, hash| + header_text = read_cell(sheet, header_row, col).to_s + year = extract_year_from_header(header_text) + hash[col] = year + end + + all_rows = (header_row + 1).upto(sheet.last_row).filter_map do |row| + line_name = read_cell(sheet, row, columns[:name_column]) + line_code = read_cell(sheet, row, columns[:code_column]).to_s.strip.presence + + values = columns[:value_columns].each_with_object({}) do |col, hash| + value = read_cell(sheet, row, col) + year = value_headers[col] + hash[year.to_s] = value if value.present? && year.present? + end.compact_blank + + next if line_name.blank? && line_code.blank? && values.blank? + + { + line_name: line_name, + line_code: line_code, + values: values + } + end + + payloads_by_year = {} + value_headers.each do |col, year| + next unless year.present? + + rows_for_year = all_rows.map do |row| + value = row[:values][year.to_s] + next unless value.present? + + { + line_name: row[:line_name], + line_code: row[:line_code], + value: value + } + end.compact + + payloads_by_year[year] = { + form_code: FORM_CODE, + sheet_name: SHEET_NAME, + header_row: header_row, + data_year: year, + rows: rows_for_year + } + end + + payloads_by_year + end + + def extract_year_from_header(header_text) + year_match = header_text[/\b(20\d{2})\b/] + year_match.to_i if year_match + end + + def find_header_row(sheet) + 1.upto(sheet.last_row) do |row| + normalized_cells = 1.upto(sheet.last_column).map { |col| normalize_label(sheet.cell(row, col)) } + has_name = normalized_cells.any? { |text| text.include?('наименование показателя') } + has_code = normalized_cells.any? { |text| text == 'код' || text.start_with?('код ') || text.start_with?('код:') } + return row if has_name && has_code + end + + 1 + end + + def detect_columns(sheet, header_row) + columns = 1.upto(sheet.last_column).each_with_object({}) do |col, hash| + hash[col] = normalize_label(sheet.cell(header_row, col)) + end + + name_column = columns.key(columns.values.find { |text| text.include?('наименование показателя') }) || 1 + + code_column = columns.key(columns.values.find { |text| + text == 'код строки' || text.start_with?('код строки ') || + (text == 'код' && columns[name_column + 1] == text) + }) || columns.key(columns.values.find { |text| text == 'код' || text.start_with?('код ') }) || [name_column + 1, sheet.last_column].min + + value_columns = columns.keys.select do |col| + text = columns[col] + text.include?('за ') || + text.include?('на ') || + text.match?(/\b20\d{2}\b/) || + (col > code_column && text.include?('года')) || + (col > code_column && text.match?(/\d+/)) + end + + if value_columns.empty? + value_columns = columns.keys.select { |col| col > code_column && columns[col].present? } + end + + { + name_column: name_column, + code_column: code_column, + value_columns: value_columns.sort.first(3) + } + end + end +end diff --git a/app/services/report_parsers/organization_info_parser.rb b/app/services/report_parsers/organization_info_parser.rb new file mode 100644 index 0000000..c9ad2d1 --- /dev/null +++ b/app/services/report_parsers/organization_info_parser.rb @@ -0,0 +1,73 @@ +module ReportParsers + class OrganizationInfoParser < BaseParser + SHEET_NAME = 'Сведения об организации' + + ORGANIZATION_LABELS = { + full_name: ['Полное наименование юридического лица'], + inn: ['ИНН'], + kpp: ['КПП'], + okpo: ['Код по ОКПО'], + okfs: ['Форма собственности (по ОКФС)'], + okopf: ['Организационно-правовая форма (по ОКОПФ)'], + okved2: ['Вид экономической деятельности по ОКВЭД 2'], + address: ['Местонахождение (адрес)'] + }.freeze + + AUDITOR_LABELS = { + name: ['Наименование аудиторской организации/ФИО индивидуального аудитора'], + inn: ['ИНН'], + ogrn: ['ОГРН/ОГРНИП'] + }.freeze + + def call + return unless sheet_exists?(SHEET_NAME) + + sheet = spreadsheet.sheet(SHEET_NAME) + parse_organization(sheet) + parse_auditor(sheet) + end + + private + + def parse_organization(sheet) + end_row = auditor_section_start_row(sheet) ? auditor_section_start_row(sheet) - 1 : sheet.last_row + org_data = ORGANIZATION_LABELS.each_with_object({}) do |(attr, labels), hash| + hash[attr] = find_value_by_labels(sheet, labels, row_range: 1..end_row) + end + + yearly_file.organization.update!(org_data) + end + + def parse_auditor(sheet) + start_row = auditor_section_start_row(sheet) || 1 + auditor_data = AUDITOR_LABELS.each_with_object({}) do |(attr, labels), hash| + hash[attr] = find_value_by_labels(sheet, labels, row_range: start_row..sheet.last_row) + end + + return if auditor_data[:name].blank? + + auditor_inn = auditor_data[:inn].to_s.strip + auditor = if auditor_inn.present? + Auditor.find_or_initialize_by(inn: auditor_inn) + else + Auditor.find_or_initialize_by(name: auditor_data[:name]) + end + + auditor.assign_attributes( + name: auditor_data[:name], + inn: auditor_inn.presence, + ogrn: auditor_data[:ogrn].to_s.strip.presence + ) + auditor.save! + + yearly_file.update!(auditor: auditor) + end + + def auditor_section_start_row(sheet) + @auditor_section_start_row ||= begin + labels = AUDITOR_LABELS.fetch(:name) + find_label_position(sheet, labels, row_range: 1..sheet.last_row)&.first + end + end + end +end diff --git a/db/migrate/20260321000006_add_unique_index_to_financial_statements.rb b/db/migrate/20260321000006_add_unique_index_to_financial_statements.rb new file mode 100644 index 0000000..9807c57 --- /dev/null +++ b/db/migrate/20260321000006_add_unique_index_to_financial_statements.rb @@ -0,0 +1,9 @@ +class AddUniqueIndexToFinancialStatements < ActiveRecord::Migration[7.1] + def change + add_index :financial_statements, + [:yearly_file_id, :document_type_id, :sheet_number], + name: 'index_fs_on_yearly_file_doc_type_sheet', + unique: true, + if_not_exists: true + end +end diff --git a/db/schema.rb b/db/schema.rb index 7cab919..7a18b93 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2026_03_12_000005) do +ActiveRecord::Schema[7.1].define(version: 2026_03_21_000006) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -65,6 +65,7 @@ ActiveRecord::Schema[7.1].define(version: 2026_03_12_000005) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["document_type_id"], name: "index_financial_statements_on_document_type_id" + t.index ["yearly_file_id", "document_type_id", "sheet_number"], name: "index_fs_on_yearly_file_doc_type_sheet", unique: true t.index ["yearly_file_id"], name: "index_financial_statements_on_yearly_file_id" end