zephyr/scripts/pylib/twister/twisterlib/size_calc.py

#!/usr/bin/env python3
# vim: set syntax=python ts=4 :
#
# Copyright (c) 2018 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

import subprocess
import sys
import os
import re
import typing
import logging
from twisterlib.error import TwisterRuntimeError

logger = logging.getLogger('twister')
logger.setLevel(logging.DEBUG)

class SizeCalculator:
    alloc_sections = [
        "bss",
        "noinit",
        "app_bss",
        "app_noinit",
        "ccm_bss",
        "ccm_noinit"
    ]

    rw_sections = [
        "datas",
        "initlevel",
        "exceptions",
        "initshell",
        "_static_thread_data_area",
        "k_timer_area",
        "k_mem_slab_area",
        "sw_isr_table",
        "k_sem_area",
        "k_mutex_area",
        "app_shmem_regions",
        "_k_fifo_area",
        "_k_lifo_area",
        "k_stack_area",
        "k_msgq_area",
        "k_mbox_area",
        "k_pipe_area",
        "net_if_area",
        "net_if_dev_area",
        "net_l2_area",
        "net_l2_data",
        "k_queue_area",
        "_net_buf_pool_area",
        "app_datas",
        "kobject_data",
        "mmu_tables",
        "app_pad",
        "priv_stacks",
        "ccm_data",
        "usb_descriptor",
        "usb_data", "usb_bos_desc",
        "uart_mux",
        'log_backends_sections',
        'log_dynamic_sections',
        'log_const_sections',
        "app_smem",
        'shell_root_cmds_sections',
        'log_const_sections',
        "font_entry_sections",
        "priv_stacks_noinit",
        "_GCOV_BSS_SECTION_NAME",
        "gcov",
        "nocache",
        "devices",
        "k_heap_area",
    ]

    # These get copied into RAM only on non-XIP
    ro_sections = [
        "rom_start",
        "text",
        "ctors",
        "init_array",
        "reset",
        "z_object_assignment_area",
        "rodata",
        "net_l2",
        "vector",
        "sw_isr_table",
        "settings_handler_static_area",
        "bt_l2cap_fixed_chan_area",
        "bt_l2cap_br_fixed_chan_area",
        "bt_gatt_service_static_area",
        "vectors",
        "net_socket_register_area",
        "net_ppp_proto",
        "shell_area",
        "tracing_backend_area",
        "ppp_protocol_handler_area",
    ]

    # Variable below is stored for calculating size using build.log
    USEFUL_LINES_AMOUNT = 4

    def __init__(self, elf_filename: str,\
        extra_sections: typing.List[str],\
        buildlog_filepath: str = '',\
        generate_warning: bool = True):
        """Constructor

        @param elf_filename (str) Path to the output binary
            parsed by objdump to determine section sizes.
        @param extra_sections (list[str]) List of extra,
            unexpected sections, which Twister should not
            report as error and not include in the
            size calculation.
        @param buildlog_filepath (str, default: '') Path to the
            output build.log
        @param generate_warning (bool, default: True) Twister should
            (or not) warning about missing the information about
            footprint.
        """
        self.elf_filename = elf_filename
        self.buildlog_filename = buildlog_filepath
        self.sections = []
        self.used_rom = 0
        self.used_ram = 0
        self.available_ram = 0
        self.available_rom = 0
        self.extra_sections = extra_sections
        self.is_xip = True
        self.generate_warning = generate_warning

        self._calculate_sizes()

    def size_report(self):
        print(self.elf_filename)
        print("SECTION NAME             VMA        LMA     SIZE  HEX SZ TYPE")
        for v in self.sections:
            print("%-17s 0x%08x 0x%08x %8d 0x%05x %-7s" %
                        (v["name"], v["virt_addr"], v["load_addr"], v["size"], v["size"],
                        v["type"]))

        print("Totals: %d bytes (ROM), %d bytes (RAM)" %
                    (self.used_rom, self.used_ram))
        print("")

    def get_used_ram(self):
        """Get the amount of RAM the application will use up on the device

        @return amount of RAM, in bytes
        """
        return self.used_ram

    def get_used_rom(self):
        """Get the size of the data that this application uses on device's flash

        @return amount of ROM, in bytes
        """
        return self.used_rom

    def unrecognized_sections(self):
        """Get a list of sections inside the binary that weren't recognized

        @return list of unrecognized section names
        """
        slist = []
        for v in self.sections:
            if not v["recognized"]:
                slist.append(v["name"])
        return slist

    def get_available_ram(self) -> int:
        """Get the total available RAM.

        @return total available RAM, in bytes (int)
        """
        return self.available_ram

    def get_available_rom(self) -> int:
        """Get the total available ROM.

        @return total available ROM, in bytes (int)
        """
        return self.available_rom

    def _calculate_sizes(self):
        """ELF file is analyzed, even if the option to read memory
        footprint from the build.log file is set.
        This is to detect potential problems contained in
        unrecognized sections of the file.
        """
        self._analyze_elf_file()
        if self.buildlog_filename.endswith("build.log"):
            self._get_footprint_from_buildlog()

    def _check_elf_file(self) -> None:
        # Make sure this is an ELF binary
        with open(self.elf_filename, "rb") as f:
            magic = f.read(4)

        try:
            if magic != b'\x7fELF':
                raise TwisterRuntimeError("%s is not an ELF binary" % self.elf_filename)
        except Exception as e:
            print(str(e))
            sys.exit(2)

    def _check_is_xip(self) -> None:
        # Search for CONFIG_XIP in the ELF's list of symbols using NM and AWK.
        # GREP can not be used as it returns an error if the symbol is not
        # found.
        is_xip_command = "nm " + self.elf_filename + \
                         " | awk '/CONFIG_XIP/ { print $3 }'"
        is_xip_output = subprocess.check_output(
            is_xip_command, shell=True, stderr=subprocess.STDOUT).decode(
            "utf-8").strip()
        try:
            if is_xip_output.endswith("no symbols"):
                raise TwisterRuntimeError("%s has no symbol information" % self.elf_filename)
        except Exception as e:
            print(str(e))
            sys.exit(2)

        self.is_xip = (len(is_xip_output) != 0)

    def _get_info_elf_sections(self) -> None:
        """Calculate RAM and ROM usage and information about issues by section"""
        objdump_command = "objdump -h " + self.elf_filename
        objdump_output = subprocess.check_output(
            objdump_command, shell=True).decode("utf-8").splitlines()

        for line in objdump_output:
            words = line.split()

            if not words:  # Skip lines that are too short
                continue

            index = words[0]
            if not index[0].isdigit():  # Skip lines that do not start
                continue  # with a digit

            name = words[1]  # Skip lines with section names
            if name[0] == '.':  # starting with '.'
                continue

            # TODO this doesn't actually reflect the size in flash or RAM as
            # it doesn't include linker-imposed padding between sections.
            # It is close though.
            size = int(words[2], 16)
            if size == 0:
                continue

            load_addr = int(words[4], 16)
            virt_addr = int(words[3], 16)

            # Add section to memory use totals (for both non-XIP and XIP scenarios)
            # Unrecognized section names are not included in the calculations.
            recognized = True

            # If build.log file exists, check errors (unrecognized sections
            # in ELF file).
            if self.buildlog_filename:
                if name in SizeCalculator.alloc_sections or\
                    SizeCalculator.rw_sections or\
                    SizeCalculator.ro_sections:
                    continue
                else:
                    stype = "unknown"
                    if name not in self.extra_sections:
                        recognized = False
            else:
                if name in SizeCalculator.alloc_sections:
                    self.used_ram += size
                    stype = "alloc"
                elif name in SizeCalculator.rw_sections:
                    self.used_ram += size
                    self.used_rom += size
                    stype = "rw"
                elif name in SizeCalculator.ro_sections:
                    self.used_rom += size
                    if not self.is_xip:
                        self.used_ram += size
                    stype = "ro"
                else:
                    stype = "unknown"
                    if name not in self.extra_sections:
                        recognized = False

            self.sections.append({"name": name, "load_addr": load_addr,
                                  "size": size, "virt_addr": virt_addr,
                                  "type": stype, "recognized": recognized})

    def _analyze_elf_file(self) -> None:
        self._check_elf_file()
        self._check_is_xip()
        self._get_info_elf_sections()

    def _get_buildlog_file_content(self) -> typing.List[str]:
        """Get content of the build.log file.

        @return Content of the build.log file (list[str])
        """
        if os.path.exists(path=self.buildlog_filename):
            with open(file=self.buildlog_filename, mode='r') as file:
                file_content = file.readlines()
        else:
            if self.generate_warning:
                logger.error(msg=f"Incorrect path to build.log file to analyze footprints. Please check the path {self.buildlog_filename}.")
            file_content = []
        return file_content

    def _find_offset_of_last_pattern_occurrence(self, file_content: typing.List[str]) -> int:
        """Find the offset from which the information about the memory footprint is read.

        @param file_content (list[str]) Content of build.log.
        @return Offset with information about the memory footprint (int)
        """
        result = -1
        if len(file_content) == 0:
            logger.warning("Build.log file is empty.")
        else:
            # Pattern to first line with information about memory footprint
            PATTERN_SEARCHED_LINE = "Memory region"
            # Check the file in reverse order.
            for idx, line in enumerate(reversed(file_content)):
                # Condition is fulfilled if the pattern matches with the start of the line.
                if re.match(pattern=PATTERN_SEARCHED_LINE, string=line):
                    result = idx + 1
                    break
        # If the file does not contain information about memory footprint, the warning is raised.
        if result == -1:
            logger.warning(msg=f"Information about memory footprint for this test configuration is not found. Please check file {self.buildlog_filename}.")
        return result

    def _get_lines_with_footprint(self, start_offset: int, file_content: typing.List[str]) -> typing.List[str]:
        """Get lines from the file with a memory footprint.

        @param start_offset (int) Offset with the first line of the information about memory footprint.
        @param file_content (list[str]) Content of the build.log file.
        @return Lines with information about memory footprint (list[str])
        """
        if len(file_content) == 0:
            result = []
        else:
            if start_offset > len(file_content) or start_offset <= 0:
                info_line_idx_start = len(file_content) - 1
            else:
                info_line_idx_start = len(file_content) - start_offset

            info_line_idx_stop = info_line_idx_start + self.USEFUL_LINES_AMOUNT
            if info_line_idx_stop > len(file_content):
                info_line_idx_stop = len(file_content)

            result = file_content[info_line_idx_start:info_line_idx_stop]
        return result

    def _clear_whitespaces_from_lines(self, text_lines: typing.List[str]) -> typing.List[str]:
        """Clear text lines from whitespaces.

        @param text_lines (list[str]) Lines with useful information.
        @return  Cleared text lines with information (list[str])
        """
        return [line.strip("\n").rstrip("%") for line in text_lines] if text_lines else []

    def _divide_text_lines_into_columns(self, text_lines: typing.List[str]) -> typing.List[typing.List[str]]:
        """Divide lines of text into columns.

        @param lines (list[list[str]]) Lines with information about memory footprint.
        @return Lines divided into columns (list[list[str]])
        """
        if text_lines:
            result = []
            PATTERN_SPLIT_COLUMNS = "  +"
            for line in text_lines:
                line = [column.rstrip(":") for column in re.split(pattern=PATTERN_SPLIT_COLUMNS, string=line)]
                result.append(list(filter(None, line)))
        else:
            result = [[]]

        return result

    def _unify_prefixes_on_all_values(self, data_lines: typing.List[typing.List[str]]) -> typing.List[typing.List[str]]:
        """Convert all values in the table to unified order of magnitude.

        @param data_lines (list[list[str]]) Lines with information about memory footprint.
        @return Lines with unified values (list[list[str]])
        """
        if len(data_lines) != self.USEFUL_LINES_AMOUNT:
            data_lines = [[]]
            if self.generate_warning:
                logger.warning(msg=f"Incomplete information about memory footprint. Please check file {self.buildlog_filename}")
        else:
            for idx, line in enumerate(data_lines):
                # Line with description of the columns
                if idx == 0:
                    continue
                line_to_replace = list(map(self._binary_prefix_converter, line))
                data_lines[idx] = line_to_replace

        return data_lines

    def _binary_prefix_converter(self, value: str) -> str:
        """Convert specific value to particular prefix.

        @param value (str) Value to convert.
        @return Converted value to output prefix (str)
        """
        PATTERN_VALUE = r"([0-9]?\s.?B\Z)"

        if not re.search(pattern=PATTERN_VALUE, string=value):
            converted_value = value.rstrip()
        else:
            PREFIX_POWER = {'B': 0, 'KB': 10, 'MB': 20, 'GB': 30}
            DEFAULT_DATA_PREFIX = 'B'

            data_parts = value.split()
            numeric_value = int(data_parts[0])
            unit = data_parts[1]
            shift = PREFIX_POWER.get(unit, 0) - PREFIX_POWER.get(DEFAULT_DATA_PREFIX, 0)
            unit_predictor = pow(2, shift)
            converted_value = str(numeric_value * unit_predictor)
        return converted_value

    def _create_data_table(self) -> typing.List[typing.List[str]]:
        """Create table with information about memory footprint.

        @return Table with information about memory usage (list[list[str]])
        """
        file_content = self._get_buildlog_file_content()
        data_line_start_idx = self._find_offset_of_last_pattern_occurrence(file_content=file_content)

        if data_line_start_idx < 0:
            data_from_content = [[]]
        else:
            # Clean lines and separate information to columns
            information_lines = self._get_lines_with_footprint(start_offset=data_line_start_idx, file_content=file_content)
            information_lines = self._clear_whitespaces_from_lines(text_lines=information_lines)
            data_from_content = self._divide_text_lines_into_columns(text_lines=information_lines)
            data_from_content = self._unify_prefixes_on_all_values(data_lines=data_from_content)

        return data_from_content

    def _get_footprint_from_buildlog(self) -> None:
        """Get memory footprint from build.log"""
        data_from_file = self._create_data_table()

        if data_from_file == [[]] or not data_from_file:
            self.used_ram = 0
            self.used_rom = 0
            self.available_ram = 0
            self.available_rom = 0
            if self.generate_warning:
                logger.warning(msg=f"Missing information about memory footprint. Check file {self.buildlog_filename}.")
        else:
            ROW_RAM_IDX = 2
            ROW_ROM_IDX = 1
            COLUMN_USED_SIZE_IDX = 1
            COLUMN_AVAILABLE_SIZE_IDX = 2

            self.used_ram = int(data_from_file[ROW_RAM_IDX][COLUMN_USED_SIZE_IDX])
            self.used_rom = int(data_from_file[ROW_ROM_IDX][COLUMN_USED_SIZE_IDX])
            self.available_ram = int(data_from_file[ROW_RAM_IDX][COLUMN_AVAILABLE_SIZE_IDX])
            self.available_rom = int(data_from_file[ROW_ROM_IDX][COLUMN_AVAILABLE_SIZE_IDX])