diff --git a/app/services/report_parser.rb b/app/services/report_parser.rb index f277c92..ccda01a 100644 --- a/app/services/report_parser.rb +++ b/app/services/report_parser.rb @@ -1,33 +1,21 @@ require 'roo' class ReportParser - SHEET_NAME = 'Сведения об организации' - CELL_MAP = { - full_name: { row: 6, col: 13 }, - inn: { row: 10, col: 13 }, - kpp: { row: 11, col: 13 }, - okpo: { row: 12, col: 13 }, - okfs: { row: 13, col: 13 }, - okopf: { row: 14, col: 13 }, - okved2: { row: 15, col: 13 }, - address: { row: 16, col: 13 } - }.freeze - - AUDITOR_MAP = { - name: { row: 19, col: 13 }, - inn: { row: 20, col: 13 }, - ogrn: { row: 21, col: 13 } - }.freeze - def initialize(yearly_file) @yearly_file = yearly_file end def call ActiveRecord::Base.transaction do - parse_organization - parse_auditor - extract_year + ReportParsers::OrganizationInfoParser.new(@yearly_file, spreadsheet: spreadsheet).call + + form_11_data = ReportParsers::Form11Parser.new(@yearly_file, spreadsheet: spreadsheet).call + form_12_data = ReportParsers::Form12Parser.new(@yearly_file, spreadsheet: spreadsheet).call + form_14_data = ReportParsers::Form14Parser.new(@yearly_file, spreadsheet: spreadsheet).call + + save_form_statements(form_11_data, 1) # 1 - Бух. баланс + save_form_statements(form_12_data, 2) # 2 - Отчет о фин. результатах + save_form_statements(form_14_data, 4) # 4 - Отчет о движении денежных средств end end @@ -43,56 +31,34 @@ class ReportParser end end - def info_sheet - @info_sheet ||= spreadsheet.sheet(SHEET_NAME) - end + def save_form_statements(form_data, form_id) + document_type = DocumentType.find_by(id: form_id) + raise "DocumentType with id=#{form_id} not found!" unless document_type - def read_cell(sheet, row, col) - value = sheet.cell(row, col) - value.is_a?(String) ? value.strip.presence : value - end + form_data.each do |year, data| + rows = data[:rows] || data['rows'] || [] + next if rows.blank? - def parse_organization - org_data = CELL_MAP.each_with_object({}) do |(attr, pos), hash| - hash[attr] = read_cell(info_sheet, pos[:row], pos[:col]) + statement = find_or_build_statement(document_type, year) + payload = { + data_year: year, + rows: rows + } + statement.assign_attributes(data: payload) + statement.save! end - - organization = @yearly_file.organization - organization.update!(org_data) - organization end - def parse_auditor - auditor_data = AUDITOR_MAP.each_with_object({}) do |(attr, pos), hash| - hash[attr] = read_cell(info_sheet, pos[:row], pos[:col]) - end + def find_or_build_statement(document_type, year) + existing = FinancialStatement + .joins(:yearly_file) + .where(document_type: document_type, sheet_number: year) + .where(yearly_files: { organization_id: @yearly_file.organization_id }) + .order('yearly_files.year DESC NULLS LAST, financial_statements.updated_at DESC') + .first - return if auditor_data[:name].blank? + return existing if existing - auditor_inn = auditor_data[:inn].to_s.strip - auditor = if auditor_inn.present? - Auditor.find_or_initialize_by(inn: auditor_inn) - else - Auditor.find_or_initialize_by(name: auditor_data[:name]) - end - - auditor.assign_attributes( - name: auditor_data[:name], - inn: auditor_inn.presence, - ogrn: auditor_data[:ogrn].to_s.strip.presence - ) - auditor.save! - - @yearly_file.update!(auditor: auditor) - auditor - end - - def extract_year - balance_sheet = spreadsheet.sheet('Бухгалтерский баланс') - period_text = balance_sheet.cell(4, 1).to_s - year = period_text.match(/(\d{4})/)&.captures&.first&.to_i - - @yearly_file.update!(year: year) if year - year + @yearly_file.financial_statements.new(document_type: document_type, sheet_number: year) end end diff --git a/app/services/report_parsers/base_parser.rb b/app/services/report_parsers/base_parser.rb new file mode 100644 index 0000000..232bc1b --- /dev/null +++ b/app/services/report_parsers/base_parser.rb @@ -0,0 +1,100 @@ +require 'set' + +module ReportParsers + class BaseParser + def initialize(yearly_file, spreadsheet:) + @yearly_file = yearly_file + @spreadsheet = spreadsheet + end + + private + + attr_reader :yearly_file, :spreadsheet + + def sheet_exists?(sheet_name) + spreadsheet.sheets.include?(sheet_name) + end + + def read_cell(sheet, row, col) + value = sheet.cell(row, col) + + # Handle merged cells: if value is nil/blank, try reading from nearby cells + if value.blank? && sheet.respond_to?(:merged_cells) + value = find_merged_cell_value(sheet, row, col) + end + + value.is_a?(String) ? value.strip.presence : value + end + + def find_merged_cell_value(sheet, row, col) + # Try to find value in merged cell range + # Check cells in a small radius around the target cell + (-1..1).each do |row_offset| + (-1..1).each do |col_offset| + next if row_offset == 0 && col_offset == 0 + + check_row = row + row_offset + check_col = col + col_offset + + next if check_row < 1 || check_col < 1 + next if check_row > sheet.last_row || check_col > sheet.last_column + + value = sheet.cell(check_row, check_col) + return value if value.present? + end + end + + nil + end + + def find_value_by_labels(sheet, labels, row_range:, value_offset: 1) + position = find_label_position(sheet, labels, row_range: row_range) + return if position.nil? + + row, col = position + + max_col = sheet.last_column.to_i + (col + value_offset).upto(max_col) do |search_col| + value = read_cell(sheet, row, search_col) + return value if value.present? + end + + nil + end + + def find_label_position(sheet, labels, row_range:) + normalized_labels = Array(labels).map { |label| normalize_label(label) }.to_set + max_col = sheet.last_column.to_i + return if max_col <= 1 + + row_range.each do |row| + 1.upto(max_col - 1) do |col| + label_text = normalize_label(sheet.cell(row, col)) + next if label_text.blank? + next unless label_matches?(label_text, normalized_labels) + + return [row, col] + end + end + + nil + end + + def label_matches?(label_text, normalized_labels) + normalized_labels.any? do |label| + label_text == label || + label_text.start_with?("#{label} ") || + label_text.start_with?("#{label}:") + end + end + + def normalize_label(value) + value.to_s + .downcase + .tr("\u00A0", ' ') + .gsub(/\s+/, ' ') + .gsub(/[«»]/, '') + .strip + end + end +end diff --git a/app/services/report_parsers/form11_parser.rb b/app/services/report_parsers/form11_parser.rb new file mode 100644 index 0000000..0185894 --- /dev/null +++ b/app/services/report_parsers/form11_parser.rb @@ -0,0 +1,128 @@ +module ReportParsers + class Form11Parser < BaseParser + SHEET_NAME = 'Бухгалтерский баланс' + FORM_CODE = '1.1' + + def call + return {} unless sheet_exists?(SHEET_NAME) + + sheet = spreadsheet.sheet(SHEET_NAME) + extract_year(sheet) + build_payloads(sheet) + end + + private + + def build_payloads(sheet) + header_row = find_header_row(sheet) + columns = detect_columns(sheet, header_row) + + + value_headers = columns[:value_columns].each_with_object({}) do |col, hash| + header_text = read_cell(sheet, header_row, col).to_s + year = extract_year_from_header(header_text) + hash[col] = year + end + + all_rows = (header_row + 1).upto(sheet.last_row).filter_map do |row| + line_name = read_cell(sheet, row, columns[:name_column]) + line_code = read_cell(sheet, row, columns[:code_column]).to_s.strip.presence + + values = columns[:value_columns].each_with_object({}) do |col, hash| + value = read_cell(sheet, row, col) + year = value_headers[col] + hash[year.to_s] = value if value.present? && year.present? + end.compact_blank + + next if line_name.blank? && line_code.blank? && values.blank? + + { + line_name: line_name, + line_code: line_code, + values: values + } + end + + payloads_by_year = {} + value_headers.each do |col, year| + next unless year.present? + + rows_for_year = all_rows.map do |row| + value = row[:values][year.to_s] + next unless value.present? + + { + line_name: row[:line_name], + line_code: row[:line_code], + value: value + } + end.compact + + payloads_by_year[year] = { + form_code: FORM_CODE, + sheet_name: SHEET_NAME, + header_row: header_row, + data_year: year, + rows: rows_for_year + } + end + + payloads_by_year + end + + def extract_year_from_header(header_text) + year_match = header_text[/\b(20\d{2})\b/] + year_match.to_i if year_match + end + + def find_header_row(sheet) + 1.upto(sheet.last_row) do |row| + normalized_cells = 1.upto(sheet.last_column).map { |col| normalize_label(sheet.cell(row, col)) } + has_name = normalized_cells.any? { |text| text.include?('наименование показателя') } + has_code = normalized_cells.any? { |text| text == 'код' || text.start_with?('код ') || text.start_with?('код:') } + return row if has_name && has_code + end + + 1 + end + + def detect_columns(sheet, header_row) + columns = 1.upto(sheet.last_column).each_with_object({}) do |col, hash| + hash[col] = normalize_label(sheet.cell(header_row, col)) + end + + name_column = columns.key(columns.values.find { |text| text.include?('наименование показателя') }) || 1 + + code_column = columns.key(columns.values.find { |text| + text == 'код строки' || text.start_with?('код строки ') || + (text == 'код' && columns[name_column + 1] == text) + }) || columns.key(columns.values.find { |text| text == 'код' || text.start_with?('код ') }) || [name_column + 1, sheet.last_column].min + + value_columns = columns.keys.select do |col| + text = columns[col] + text.include?('на 31') || + text.include?('на ') || + text.match?(/\bна \d+ /) || + text.match?(/\b20\d{2}\b/) || + (col > code_column && text.include?('декабря')) || + (col > code_column && text.match?(/\d+/)) # fallback: numeric values + end + + if value_columns.empty? + value_columns = columns.keys.select { |col| col > code_column && columns[col].present? } + end + + { + name_column: name_column, + code_column: code_column, + value_columns: value_columns.sort.first(3) + } + end + + def extract_year(sheet) + period_text = sheet.cell(4, 1).to_s + year = period_text[/\d{4}/].to_i + yearly_file.update!(year: year) if year.positive? + end + end +end diff --git a/app/services/report_parsers/form12_parser.rb b/app/services/report_parsers/form12_parser.rb new file mode 100644 index 0000000..90f366b --- /dev/null +++ b/app/services/report_parsers/form12_parser.rb @@ -0,0 +1,114 @@ +module ReportParsers + class Form12Parser < BaseParser + SHEET_NAME = 'Отчет о финансовых результатах' + FORM_CODE = '1.2' + + def call + return {} unless sheet_exists?(SHEET_NAME) + + sheet = spreadsheet.sheet(SHEET_NAME) + build_payloads(sheet) + end + + private + + def build_payloads(sheet) + header_row = find_header_row(sheet) + columns = detect_columns(sheet, header_row) + + value_headers = columns[:value_columns].each_with_object({}) do |col, hash| + header_text = read_cell(sheet, header_row, col).to_s + year = extract_year_from_header(header_text) + hash[col] = year + end + + all_rows = (header_row + 1).upto(sheet.last_row).filter_map do |row| + line_name = read_cell(sheet, row, columns[:name_column]) + line_code = read_cell(sheet, row, columns[:code_column]).to_s.strip.presence + + values = columns[:value_columns].each_with_object({}) do |col, hash| + value = read_cell(sheet, row, col) + year = value_headers[col] + hash[year.to_s] = value if value.present? && year.present? + end.compact_blank + + next if line_name.blank? && line_code.blank? && values.blank? + + { + line_name: line_name, + line_code: line_code, + values: values + } + end + + payloads_by_year = {} + value_headers.each do |col, year| + next unless year.present? + + rows_for_year = all_rows.map do |row| + value = row[:values][year.to_s] + next unless value.present? + + { + line_name: row[:line_name], + line_code: row[:line_code], + value: value + } + end.compact + + payloads_by_year[year] = { + form_code: FORM_CODE, + sheet_name: SHEET_NAME, + header_row: header_row, + data_year: year, + rows: rows_for_year + } + end + + payloads_by_year + end + + def extract_year_from_header(header_text) + year_match = header_text[/\b(20\d{2})\b/] + year_match.to_i if year_match + end + + def find_header_row(sheet) + 1.upto(sheet.last_row) do |row| + normalized_cells = 1.upto(sheet.last_column).map { |col| normalize_label(sheet.cell(row, col)) } + has_name = normalized_cells.any? { |text| text.include?('наименование показателя') } + has_code = normalized_cells.any? { |text| text == 'код' || text.start_with?('код ') || text.start_with?('код:') } + return row if has_name && has_code + end + + 1 + end + + def detect_columns(sheet, header_row) + columns = 1.upto(sheet.last_column).each_with_object({}) do |col, hash| + hash[col] = normalize_label(sheet.cell(header_row, col)) + end + + name_column = columns.key(columns.values.find { |text| text.include?('наименование показателя') }) || 1 + + code_column = columns.key(columns.values.find { |text| + text == 'код строки' || text.start_with?('код строки ') || + (text == 'код' && columns[name_column + 1] == text) + }) || columns.key(columns.values.find { |text| text == 'код' || text.start_with?('код ') }) || [name_column + 1, sheet.last_column].min + + value_columns = columns.keys.select do |col| + text = columns[col] + next false if text.include?('поясн') + text.match?(/20\d{2}/) || text.include?('за ') || text.include?('на ') + end + + value_columns = value_columns.sort.last(2) + + { + name_column: name_column, + code_column: code_column, + value_columns: value_columns + } + end + end +end diff --git a/app/services/report_parsers/form14_parser.rb b/app/services/report_parsers/form14_parser.rb new file mode 100644 index 0000000..0c935bf --- /dev/null +++ b/app/services/report_parsers/form14_parser.rb @@ -0,0 +1,112 @@ + def extract_year_from_header(header_text) + year_match = header_text[/\b(20\d{2})\b/] + year_match.to_i if year_match + end + def find_header_row(sheet) + 1.upto(sheet.last_row) do |row| + normalized_cells = 1.upto(sheet.last_column).map { |col| normalize_label(sheet.cell(row, col)) } + has_name = normalized_cells.any? { |text| text.include?("наименование показателя") } + has_code = normalized_cells.any? { |text| text == "код" || text.start_with?("код ") || text.start_with?("код:") } + return row if has_name && has_code + end + + 1 + end + + def detect_columns(sheet, header_row) + columns = 1.upto(sheet.last_column).each_with_object({}) do |col, hash| + hash[col] = normalize_label(sheet.cell(header_row, col)) + end + + name_column = columns.key(columns.values.find { |text| text.include?("наименование показателя") }) || 1 + + code_column = columns.key(columns.values.find { |text| + text == "код строки" || text.start_with?("код строки ") || + (text == "код" && columns[name_column + 1] == text) + }) || columns.key(columns.values.find { |text| text == "код" || text.start_with?("код ") }) || [name_column + 1, sheet.last_column].min + + value_columns = columns.keys.select do |col| + text = columns[col] + next false if text.include?("поясн") + text.match?(/20\d{2}/) || text.include?("за ") || text.include?("на ") + end + value_columns = value_columns.sort.last(2) + + { + name_column: name_column, + code_column: code_column, + value_columns: value_columns + } + end + + def normalize_label(text) + text.to_s.downcase.gsub(/[[:space:]]+/, ' ').strip + end +module ReportParsers + class Form14Parser < BaseParser + SHEET_NAME = 'Отчет о движении денежных средс' + FORM_CODE = '1.4' + + def call + return {} unless sheet_exists?(SHEET_NAME) + + sheet = spreadsheet.sheet(SHEET_NAME) + build_payloads(sheet) + end + + private + + def build_payloads(sheet) + header_row = find_header_row(sheet) + columns = detect_columns(sheet, header_row) + + value_headers = columns[:value_columns].each_with_object({}) do |col, hash| + header_text = read_cell(sheet, header_row, col).to_s + year = extract_year_from_header(header_text) + hash[col] = year + end + + all_rows = (header_row + 1).upto(sheet.last_row).filter_map do |row| + line_name = read_cell(sheet, row, columns[:name_column]) + line_code = read_cell(sheet, row, columns[:code_column]).to_s.strip.presence + + values = columns[:value_columns].each_with_object({}) do |col, hash| + value = read_cell(sheet, row, col) + year = value_headers[col] + hash[year.to_s] = value if value.present? && year.present? + end.compact_blank + + next if line_name.blank? && line_code.blank? && values.blank? + + { + line_name: line_name, + line_code: line_code, + values: values + } + end + + payloads_by_year = {} + value_headers.each do |col, year| + next unless year.present? + + rows_for_year = all_rows.map do |row| + value = row[:values][year.to_s] + next unless value.present? + + { + line_name: row[:line_name], + line_code: row[:line_code], + value: value + } + end.compact + + payloads_by_year[year] = { + form_code: FORM_CODE, + rows: rows_for_year + } + end + + payloads_by_year + end + end +end diff --git a/app/services/report_parsers/organization_info_parser.rb b/app/services/report_parsers/organization_info_parser.rb new file mode 100644 index 0000000..c9ad2d1 --- /dev/null +++ b/app/services/report_parsers/organization_info_parser.rb @@ -0,0 +1,73 @@ +module ReportParsers + class OrganizationInfoParser < BaseParser + SHEET_NAME = 'Сведения об организации' + + ORGANIZATION_LABELS = { + full_name: ['Полное наименование юридического лица'], + inn: ['ИНН'], + kpp: ['КПП'], + okpo: ['Код по ОКПО'], + okfs: ['Форма собственности (по ОКФС)'], + okopf: ['Организационно-правовая форма (по ОКОПФ)'], + okved2: ['Вид экономической деятельности по ОКВЭД 2'], + address: ['Местонахождение (адрес)'] + }.freeze + + AUDITOR_LABELS = { + name: ['Наименование аудиторской организации/ФИО индивидуального аудитора'], + inn: ['ИНН'], + ogrn: ['ОГРН/ОГРНИП'] + }.freeze + + def call + return unless sheet_exists?(SHEET_NAME) + + sheet = spreadsheet.sheet(SHEET_NAME) + parse_organization(sheet) + parse_auditor(sheet) + end + + private + + def parse_organization(sheet) + end_row = auditor_section_start_row(sheet) ? auditor_section_start_row(sheet) - 1 : sheet.last_row + org_data = ORGANIZATION_LABELS.each_with_object({}) do |(attr, labels), hash| + hash[attr] = find_value_by_labels(sheet, labels, row_range: 1..end_row) + end + + yearly_file.organization.update!(org_data) + end + + def parse_auditor(sheet) + start_row = auditor_section_start_row(sheet) || 1 + auditor_data = AUDITOR_LABELS.each_with_object({}) do |(attr, labels), hash| + hash[attr] = find_value_by_labels(sheet, labels, row_range: start_row..sheet.last_row) + end + + return if auditor_data[:name].blank? + + auditor_inn = auditor_data[:inn].to_s.strip + auditor = if auditor_inn.present? + Auditor.find_or_initialize_by(inn: auditor_inn) + else + Auditor.find_or_initialize_by(name: auditor_data[:name]) + end + + auditor.assign_attributes( + name: auditor_data[:name], + inn: auditor_inn.presence, + ogrn: auditor_data[:ogrn].to_s.strip.presence + ) + auditor.save! + + yearly_file.update!(auditor: auditor) + end + + def auditor_section_start_row(sheet) + @auditor_section_start_row ||= begin + labels = AUDITOR_LABELS.fetch(:name) + find_label_position(sheet, labels, row_range: 1..sheet.last_row)&.first + end + end + end +end diff --git a/db/migrate/20260321000006_add_unique_index_to_financial_statements.rb b/db/migrate/20260321000006_add_unique_index_to_financial_statements.rb new file mode 100644 index 0000000..9807c57 --- /dev/null +++ b/db/migrate/20260321000006_add_unique_index_to_financial_statements.rb @@ -0,0 +1,9 @@ +class AddUniqueIndexToFinancialStatements < ActiveRecord::Migration[7.1] + def change + add_index :financial_statements, + [:yearly_file_id, :document_type_id, :sheet_number], + name: 'index_fs_on_yearly_file_doc_type_sheet', + unique: true, + if_not_exists: true + end +end diff --git a/db/migrate/20260325063000_seed_document_types.rb b/db/migrate/20260325063000_seed_document_types.rb new file mode 100644 index 0000000..3f8724a --- /dev/null +++ b/db/migrate/20260325063000_seed_document_types.rb @@ -0,0 +1,18 @@ +class SeedDocumentTypes < ActiveRecord::Migration[7.0] + def up + execute <<-SQL + INSERT INTO document_types (id, name, created_at, updated_at) VALUES + (1, 'Бухгалтерский баланс', NOW(), NOW()), + (2, 'Отчет о финансовых результатах', NOW(), NOW()), + (3, 'Отчет об изменениях капитала', NOW(), NOW()), + (4, 'Отчет о движении денежных средств', NOW(), NOW()) + ON CONFLICT (id) DO NOTHING; + SQL + end + + def down + execute <<-SQL + DELETE FROM document_types WHERE id IN (1,2,3,4); + SQL + end +end diff --git a/db/schema.rb b/db/schema.rb index 7cab919..72e9e6b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2026_03_12_000005) do +ActiveRecord::Schema[7.1].define(version: 2026_03_25_063000) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -65,6 +65,7 @@ ActiveRecord::Schema[7.1].define(version: 2026_03_12_000005) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.index ["document_type_id"], name: "index_financial_statements_on_document_type_id" + t.index ["yearly_file_id", "document_type_id", "sheet_number"], name: "index_fs_on_yearly_file_doc_type_sheet", unique: true t.index ["yearly_file_id"], name: "index_financial_statements_on_yearly_file_id" end