Feature/add memap cstack usage ports (#661)
* Added memap, avstack, and checkstackusage tools to STM32F4xx Makefile and CMake builds to calculate CSTACK depth and RAM usage * Added memap, cstack, and ram-usage recipes to stm32f10x port Makefile. Added Cmake build. * Removed local dlmstp.c module from stm32f10x port, and used the common datalink dlmstp.c module with MS/TP extended frames and zero-config support. * Added .nm and .su to .gitignore to skip the analysis file residue.
This commit is contained in:
Steve Karg
Executable
+979
@@ -0,0 +1,979 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""
|
||||
Copyright (c) 2016-2019 ARM Limited. All rights reserved.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
from __future__ import print_function, division, absolute_import
|
||||
|
||||
from abc import abstractmethod, ABCMeta
|
||||
from sys import stdout, exit, argv, path
|
||||
from os import sep
|
||||
from os.path import (basename, dirname, join, relpath, abspath, commonprefix,
|
||||
splitext)
|
||||
import re
|
||||
import csv
|
||||
import json
|
||||
from argparse import ArgumentParser
|
||||
from copy import deepcopy
|
||||
from collections import defaultdict
|
||||
from prettytable import PrettyTable, HEADER
|
||||
from jinja2 import FileSystemLoader, StrictUndefined
|
||||
from jinja2.environment import Environment
|
||||
from future.utils import with_metaclass
|
||||
|
||||
|
||||
# Be sure that the tools directory is in the search path
|
||||
ROOT = abspath(join(dirname(__file__), ".."))
|
||||
path.insert(0, ROOT)
|
||||
|
||||
from utils import (
|
||||
argparse_filestring_type,
|
||||
argparse_lowercase_hyphen_type,
|
||||
argparse_uppercase_type
|
||||
) # noqa: E402
|
||||
|
||||
|
||||
class _Parser(with_metaclass(ABCMeta, object)):
|
||||
"""Internal interface for parsing"""
|
||||
SECTIONS = ('.text', '.data', '.bss', '.heap', '.stack')
|
||||
MISC_FLASH_SECTIONS = ('.interrupts', '.flash_config')
|
||||
OTHER_SECTIONS = ('.interrupts_ram', '.init', '.ARM.extab',
|
||||
'.ARM.exidx', '.ARM.attributes', '.eh_frame',
|
||||
'.init_array', '.fini_array', '.jcr', '.stab',
|
||||
'.stabstr', '.ARM.exidx', '.ARM')
|
||||
|
||||
def __init__(self):
|
||||
self.modules = dict()
|
||||
|
||||
def module_add(self, object_name, size, section):
|
||||
""" Adds a module or section to the list
|
||||
|
||||
Positional arguments:
|
||||
object_name - name of the entry to add
|
||||
size - the size of the module being added
|
||||
section - the section the module contributes to
|
||||
"""
|
||||
if not object_name or not size or not section:
|
||||
return
|
||||
|
||||
if object_name in self.modules:
|
||||
self.modules[object_name].setdefault(section, 0)
|
||||
self.modules[object_name][section] += size
|
||||
return
|
||||
|
||||
obj_split = sep + basename(object_name)
|
||||
for module_path, contents in self.modules.items():
|
||||
if module_path.endswith(obj_split) or module_path == object_name:
|
||||
contents.setdefault(section, 0)
|
||||
contents[section] += size
|
||||
return
|
||||
|
||||
new_module = defaultdict(int)
|
||||
new_module[section] = size
|
||||
self.modules[object_name] = new_module
|
||||
|
||||
def module_replace(self, old_object, new_object):
|
||||
""" Replaces an object name with a new one
|
||||
"""
|
||||
if old_object in self.modules:
|
||||
self.modules[new_object] = self.modules[old_object]
|
||||
del self.modules[old_object]
|
||||
|
||||
@abstractmethod
|
||||
def parse_mapfile(self, mapfile):
|
||||
"""Parse a given file object pointing to a map file
|
||||
|
||||
Positional arguments:
|
||||
mapfile - an open file object that reads a map file
|
||||
|
||||
return value - a dict mapping from object names to section dicts,
|
||||
where a section dict maps from sections to sizes
|
||||
"""
|
||||
raise NotImplemented
|
||||
|
||||
|
||||
class _GccParser(_Parser):
|
||||
RE_OBJECT_FILE = re.compile(r'^(.+\/.+\.o(bj)?)$')
|
||||
RE_LIBRARY_OBJECT = re.compile(
|
||||
r'^.+' + r''.format(sep) + r'lib((.+\.a)\((.+\.o(bj)?)\))$'
|
||||
)
|
||||
RE_STD_SECTION = re.compile(r'^\s+.*0x(\w{8,16})\s+0x(\w+)\s(.+)$')
|
||||
RE_FILL_SECTION = re.compile(r'^\s*\*fill\*\s+0x(\w{8,16})\s+0x(\w+).*$')
|
||||
RE_TRANS_FILE = re.compile(r'^(.+\/|.+\.ltrans.o(bj)?)$')
|
||||
OBJECT_EXTENSIONS = (".o", ".obj")
|
||||
|
||||
ALL_SECTIONS = (
|
||||
_Parser.SECTIONS
|
||||
+ _Parser.OTHER_SECTIONS
|
||||
+ _Parser.MISC_FLASH_SECTIONS
|
||||
+ ('unknown', 'OUTPUT')
|
||||
)
|
||||
|
||||
def check_new_section(self, line):
|
||||
""" Check whether a new section in a map file has been detected
|
||||
|
||||
Positional arguments:
|
||||
line - the line to check for a new section
|
||||
|
||||
return value - A section name, if a new section was found, None
|
||||
otherwise
|
||||
"""
|
||||
line_s = line.strip()
|
||||
for i in self.ALL_SECTIONS:
|
||||
if line_s.startswith(i):
|
||||
return i
|
||||
if line.startswith('.'):
|
||||
return 'unknown'
|
||||
else:
|
||||
return None
|
||||
|
||||
def parse_object_name(self, line):
|
||||
""" Parse a path to object file
|
||||
|
||||
Positional arguments:
|
||||
line - the path to parse the object and module name from
|
||||
|
||||
return value - an object file name
|
||||
"""
|
||||
if re.match(self.RE_TRANS_FILE, line):
|
||||
return '[misc]'
|
||||
|
||||
test_re_mbed_os_name = re.match(self.RE_OBJECT_FILE, line)
|
||||
|
||||
if test_re_mbed_os_name:
|
||||
object_name = test_re_mbed_os_name.group(1)
|
||||
|
||||
# corner case: certain objects are provided by the GCC toolchain
|
||||
if 'arm-none-eabi' in line:
|
||||
return join('[lib]', 'misc', basename(object_name))
|
||||
return object_name
|
||||
|
||||
else:
|
||||
test_re_obj_name = re.match(self.RE_LIBRARY_OBJECT, line)
|
||||
|
||||
if test_re_obj_name:
|
||||
return join('[lib]', test_re_obj_name.group(2),
|
||||
test_re_obj_name.group(3))
|
||||
else:
|
||||
if (
|
||||
not line.startswith("LONG") and
|
||||
not line.startswith("linker stubs")
|
||||
):
|
||||
print("Unknown object name found in GCC map file: %s"
|
||||
% line)
|
||||
return '[misc]'
|
||||
|
||||
def parse_section(self, line):
|
||||
""" Parse data from a section of gcc map file
|
||||
|
||||
examples:
|
||||
0x00004308 0x7c ./BUILD/K64F/GCC_ARM/spi_api.o
|
||||
.text 0x00000608 0x198 ./BUILD/K64F/HAL_CM4.o
|
||||
|
||||
Positional arguments:
|
||||
line - the line to parse a section from
|
||||
"""
|
||||
is_fill = re.match(self.RE_FILL_SECTION, line)
|
||||
if is_fill:
|
||||
o_name = '[fill]'
|
||||
o_size = int(is_fill.group(2), 16)
|
||||
return [o_name, o_size]
|
||||
|
||||
is_section = re.match(self.RE_STD_SECTION, line)
|
||||
if is_section:
|
||||
o_size = int(is_section.group(2), 16)
|
||||
if o_size:
|
||||
o_name = self.parse_object_name(is_section.group(3))
|
||||
return [o_name, o_size]
|
||||
|
||||
return ["", 0]
|
||||
|
||||
def parse_mapfile(self, file_desc):
|
||||
""" Main logic to decode gcc map files
|
||||
|
||||
Positional arguments:
|
||||
file_desc - a stream object to parse as a gcc map file
|
||||
"""
|
||||
current_section = 'unknown'
|
||||
|
||||
with file_desc as infile:
|
||||
for line in infile:
|
||||
if line.startswith('Linker script and memory map'):
|
||||
current_section = "unknown"
|
||||
break
|
||||
|
||||
for line in infile:
|
||||
next_section = self.check_new_section(line)
|
||||
|
||||
if next_section == "OUTPUT":
|
||||
break
|
||||
elif next_section:
|
||||
current_section = next_section
|
||||
|
||||
object_name, object_size = self.parse_section(line)
|
||||
self.module_add(object_name, object_size, current_section)
|
||||
|
||||
common_prefix = dirname(commonprefix([
|
||||
o for o in self.modules.keys()
|
||||
if (
|
||||
o.endswith(self.OBJECT_EXTENSIONS)
|
||||
and not o.startswith("[lib]")
|
||||
)]))
|
||||
new_modules = {}
|
||||
for name, stats in self.modules.items():
|
||||
if name.startswith("[lib]"):
|
||||
new_modules[name] = stats
|
||||
elif name.endswith(self.OBJECT_EXTENSIONS):
|
||||
new_modules[relpath(name, common_prefix)] = stats
|
||||
else:
|
||||
new_modules[name] = stats
|
||||
return new_modules
|
||||
|
||||
|
||||
class _ArmccParser(_Parser):
|
||||
RE = re.compile(
|
||||
r'^\s+0x(\w{8})\s+0x(\w{8})\s+(\w+)\s+(\w+)\s+(\d+)\s+[*]?.+\s+(.+)$')
|
||||
RE_OBJECT = re.compile(r'(.+\.(l|a|ar))\((.+\.o(bj)?)\)')
|
||||
OBJECT_EXTENSIONS = (".o", ".obj")
|
||||
|
||||
def parse_object_name(self, line):
|
||||
""" Parse object file
|
||||
|
||||
Positional arguments:
|
||||
line - the line containing the object or library
|
||||
"""
|
||||
if line.endswith(self.OBJECT_EXTENSIONS):
|
||||
return line
|
||||
|
||||
else:
|
||||
is_obj = re.match(self.RE_OBJECT, line)
|
||||
if is_obj:
|
||||
return join(
|
||||
'[lib]', basename(is_obj.group(1)), is_obj.group(3)
|
||||
)
|
||||
else:
|
||||
print(
|
||||
"Malformed input found when parsing ARMCC map: %s" % line
|
||||
)
|
||||
return '[misc]'
|
||||
|
||||
def parse_section(self, line):
|
||||
""" Parse data from an armcc map file
|
||||
|
||||
Examples of armcc map file:
|
||||
Base_Addr Size Type Attr Idx E Section Name Object
|
||||
0x00000000 0x00000400 Data RO 11222 self.RESET startup_MK64F12.o
|
||||
0x00000410 0x00000008 Code RO 49364 * !!!main c_w.l(__main.o)
|
||||
|
||||
Positional arguments:
|
||||
line - the line to parse the section data from
|
||||
""" # noqa: E501
|
||||
test_re = re.match(self.RE, line)
|
||||
|
||||
if (
|
||||
test_re
|
||||
and "ARM_LIB_HEAP" not in line
|
||||
):
|
||||
size = int(test_re.group(2), 16)
|
||||
|
||||
if test_re.group(4) == 'RO':
|
||||
section = '.text'
|
||||
else:
|
||||
if test_re.group(3) == 'Data':
|
||||
section = '.data'
|
||||
elif test_re.group(3) == 'Zero':
|
||||
section = '.bss'
|
||||
elif test_re.group(3) == 'Code':
|
||||
section = '.text'
|
||||
else:
|
||||
print(
|
||||
"Malformed input found when parsing armcc map: %s, %r"
|
||||
% (line, test_re.groups())
|
||||
)
|
||||
|
||||
return ["", 0, ""]
|
||||
|
||||
# check name of object or library
|
||||
object_name = self.parse_object_name(
|
||||
test_re.group(6))
|
||||
|
||||
return [object_name, size, section]
|
||||
|
||||
else:
|
||||
return ["", 0, ""]
|
||||
|
||||
def parse_mapfile(self, file_desc):
|
||||
""" Main logic to decode armc5 map files
|
||||
|
||||
Positional arguments:
|
||||
file_desc - a file like object to parse as an armc5 map file
|
||||
"""
|
||||
with file_desc as infile:
|
||||
# Search area to parse
|
||||
for line in infile:
|
||||
if line.startswith(' Base Addr Size'):
|
||||
break
|
||||
|
||||
# Start decoding the map file
|
||||
for line in infile:
|
||||
self.module_add(*self.parse_section(line))
|
||||
|
||||
common_prefix = dirname(commonprefix([
|
||||
o for o in self.modules.keys()
|
||||
if (
|
||||
o.endswith(self.OBJECT_EXTENSIONS)
|
||||
and o != "anon$$obj.o"
|
||||
and o != "anon$$obj.obj"
|
||||
and not o.startswith("[lib]")
|
||||
)]))
|
||||
new_modules = {}
|
||||
for name, stats in self.modules.items():
|
||||
if (
|
||||
name == "anon$$obj.o"
|
||||
or name == "anon$$obj.obj"
|
||||
or name.startswith("[lib]")
|
||||
):
|
||||
new_modules[name] = stats
|
||||
elif name.endswith(self.OBJECT_EXTENSIONS):
|
||||
new_modules[relpath(name, common_prefix)] = stats
|
||||
else:
|
||||
new_modules[name] = stats
|
||||
return new_modules
|
||||
|
||||
|
||||
class _IarParser(_Parser):
|
||||
RE = re.compile(
|
||||
r'^\s+(.+)\s+(zero|const|ro code|inited|uninit)\s'
|
||||
r'+0x([\'\w]+)\s+0x(\w+)\s+(.+)\s.+$')
|
||||
|
||||
RE_CMDLINE_FILE = re.compile(r'^#\s+(.+\.o(bj)?)')
|
||||
RE_LIBRARY = re.compile(r'^(.+\.a)\:.+$')
|
||||
RE_OBJECT_LIBRARY = re.compile(r'^\s+(.+\.o(bj)?)\s.*')
|
||||
OBJECT_EXTENSIONS = (".o", ".obj")
|
||||
|
||||
def __init__(self):
|
||||
_Parser.__init__(self)
|
||||
# Modules passed to the linker on the command line
|
||||
# this is a dict because modules are looked up by their basename
|
||||
self.cmd_modules = {}
|
||||
|
||||
def parse_object_name(self, object_name):
|
||||
""" Parse object file
|
||||
|
||||
Positional arguments:
|
||||
line - the line containing the object or library
|
||||
"""
|
||||
if object_name.endswith(self.OBJECT_EXTENSIONS):
|
||||
try:
|
||||
return self.cmd_modules[object_name]
|
||||
except KeyError:
|
||||
return object_name
|
||||
else:
|
||||
return '[misc]'
|
||||
|
||||
def parse_section(self, line):
|
||||
""" Parse data from an IAR map file
|
||||
|
||||
Examples of IAR map file:
|
||||
Section Kind Address Size Object
|
||||
.intvec ro code 0x00000000 0x198 startup_MK64F12.o [15]
|
||||
.rodata const 0x00000198 0x0 zero_init3.o [133]
|
||||
.iar.init_table const 0x00008384 0x2c - Linker created -
|
||||
Initializer bytes const 0x00000198 0xb2 <for P3 s0>
|
||||
.data inited 0x20000000 0xd4 driverAtmelRFInterface.o [70]
|
||||
.bss zero 0x20000598 0x318 RTX_Conf_CM.o [4]
|
||||
.iar.dynexit uninit 0x20001448 0x204 <Block tail>
|
||||
HEAP uninit 0x20001650 0x10000 <Block tail>
|
||||
|
||||
Positional_arguments:
|
||||
line - the line to parse section data from
|
||||
""" # noqa: E501
|
||||
test_re = re.match(self.RE, line)
|
||||
if test_re:
|
||||
if (
|
||||
test_re.group(2) == 'const' or
|
||||
test_re.group(2) == 'ro code'
|
||||
):
|
||||
section = '.text'
|
||||
elif (test_re.group(2) == 'zero' or
|
||||
test_re.group(2) == 'uninit'):
|
||||
if test_re.group(1)[0:4] == 'HEAP':
|
||||
section = '.heap'
|
||||
elif test_re.group(1)[0:6] == 'CSTACK':
|
||||
section = '.stack'
|
||||
else:
|
||||
section = '.bss' # default section
|
||||
|
||||
elif test_re.group(2) == 'inited':
|
||||
section = '.data'
|
||||
else:
|
||||
print("Malformed input found when parsing IAR map: %s" % line)
|
||||
return ["", 0, ""]
|
||||
|
||||
# lookup object in dictionary and return module name
|
||||
object_name = self.parse_object_name(test_re.group(5))
|
||||
|
||||
size = int(test_re.group(4), 16)
|
||||
return [object_name, size, section]
|
||||
|
||||
else:
|
||||
return ["", 0, ""]
|
||||
|
||||
def check_new_library(self, line):
|
||||
"""
|
||||
Searches for libraries and returns name. Example:
|
||||
m7M_tls.a: [43]
|
||||
|
||||
"""
|
||||
test_address_line = re.match(self.RE_LIBRARY, line)
|
||||
if test_address_line:
|
||||
return test_address_line.group(1)
|
||||
else:
|
||||
return ""
|
||||
|
||||
def check_new_object_lib(self, line):
|
||||
"""
|
||||
Searches for objects within a library section and returns name.
|
||||
Example:
|
||||
rt7M_tl.a: [44]
|
||||
ABImemclr4.o 6
|
||||
ABImemcpy_unaligned.o 118
|
||||
ABImemset48.o 50
|
||||
I64DivMod.o 238
|
||||
I64DivZer.o 2
|
||||
|
||||
"""
|
||||
test_address_line = re.match(self.RE_OBJECT_LIBRARY, line)
|
||||
if test_address_line:
|
||||
return test_address_line.group(1)
|
||||
else:
|
||||
return ""
|
||||
|
||||
def parse_command_line(self, lines):
|
||||
"""Parse the files passed on the command line to the iar linker
|
||||
|
||||
Positional arguments:
|
||||
lines -- an iterator over the lines within a file
|
||||
"""
|
||||
for line in lines:
|
||||
if line.startswith("*"):
|
||||
break
|
||||
for arg in line.split(" "):
|
||||
arg = arg.rstrip(" \n")
|
||||
if (
|
||||
not arg.startswith("-")
|
||||
and arg.endswith(self.OBJECT_EXTENSIONS)
|
||||
):
|
||||
self.cmd_modules[basename(arg)] = arg
|
||||
|
||||
common_prefix = dirname(commonprefix(list(self.cmd_modules.values())))
|
||||
self.cmd_modules = {s: relpath(f, common_prefix)
|
||||
for s, f in self.cmd_modules.items()}
|
||||
|
||||
def parse_mapfile(self, file_desc):
|
||||
""" Main logic to decode IAR map files
|
||||
|
||||
Positional arguments:
|
||||
file_desc - a file like object to parse as an IAR map file
|
||||
"""
|
||||
with file_desc as infile:
|
||||
self.parse_command_line(infile)
|
||||
|
||||
for line in infile:
|
||||
if line.startswith(' Section '):
|
||||
break
|
||||
|
||||
for line in infile:
|
||||
self.module_add(*self.parse_section(line))
|
||||
|
||||
if line.startswith('*** MODULE SUMMARY'): # finish section
|
||||
break
|
||||
|
||||
current_library = ""
|
||||
for line in infile:
|
||||
library = self.check_new_library(line)
|
||||
|
||||
if library:
|
||||
current_library = library
|
||||
|
||||
object_name = self.check_new_object_lib(line)
|
||||
|
||||
if object_name and current_library:
|
||||
temp = join('[lib]', current_library, object_name)
|
||||
self.module_replace(object_name, temp)
|
||||
return self.modules
|
||||
|
||||
|
||||
class MemapParser(object):
|
||||
"""An object that represents parsed results, parses the memory map files,
|
||||
and writes out different file types of memory results
|
||||
"""
|
||||
|
||||
print_sections = ('.text', '.data', '.bss')
|
||||
delta_sections = ('.text-delta', '.data-delta', '.bss-delta')
|
||||
|
||||
# sections to print info (generic for all toolchains)
|
||||
sections = _Parser.SECTIONS
|
||||
misc_flash_sections = _Parser.MISC_FLASH_SECTIONS
|
||||
other_sections = _Parser.OTHER_SECTIONS
|
||||
|
||||
def __init__(self):
|
||||
# list of all modules and their sections
|
||||
# full list - doesn't change with depth
|
||||
self.modules = dict()
|
||||
self.old_modules = None
|
||||
# short version with specific depth
|
||||
self.short_modules = dict()
|
||||
|
||||
# Memory report (sections + summary)
|
||||
self.mem_report = []
|
||||
|
||||
# Memory summary
|
||||
self.mem_summary = dict()
|
||||
|
||||
# Totals of ".text", ".data" and ".bss"
|
||||
self.subtotal = dict()
|
||||
|
||||
# Flash no associated with a module
|
||||
self.misc_flash_mem = 0
|
||||
|
||||
# Name of the toolchain, for better headings
|
||||
self.tc_name = None
|
||||
|
||||
def reduce_depth(self, depth):
|
||||
"""
|
||||
populates the short_modules attribute with a truncated module list
|
||||
|
||||
(1) depth = 1:
|
||||
main.o
|
||||
mbed-os
|
||||
|
||||
(2) depth = 2:
|
||||
main.o
|
||||
mbed-os/test.o
|
||||
mbed-os/drivers
|
||||
|
||||
"""
|
||||
if depth == 0 or depth is None:
|
||||
self.short_modules = deepcopy(self.modules)
|
||||
else:
|
||||
self.short_modules = dict()
|
||||
for module_name, v in self.modules.items():
|
||||
split_name = module_name.split(sep)
|
||||
if split_name[0] == '':
|
||||
split_name = split_name[1:]
|
||||
new_name = join(*split_name[:depth])
|
||||
self.short_modules.setdefault(new_name, defaultdict(int))
|
||||
for section_idx, value in v.items():
|
||||
self.short_modules[new_name][section_idx] += value
|
||||
delta_name = section_idx + '-delta'
|
||||
self.short_modules[new_name][delta_name] += value
|
||||
if self.old_modules:
|
||||
for module_name, v in self.old_modules.items():
|
||||
split_name = module_name.split(sep)
|
||||
if split_name[0] == '':
|
||||
split_name = split_name[1:]
|
||||
new_name = join(*split_name[:depth])
|
||||
self.short_modules.setdefault(new_name, defaultdict(int))
|
||||
for section_idx, value in v.items():
|
||||
delta_name = section_idx + '-delta'
|
||||
self.short_modules[new_name][delta_name] -= value
|
||||
|
||||
export_formats = ["json", "csv-ci", "html", "table"]
|
||||
|
||||
def generate_output(self, export_format, depth, file_output=None):
|
||||
""" Generates summary of memory map data
|
||||
|
||||
Positional arguments:
|
||||
export_format - the format to dump
|
||||
|
||||
Keyword arguments:
|
||||
file_desc - descriptor (either stdout or file)
|
||||
depth - directory depth on report
|
||||
|
||||
Returns: generated string for the 'table' format, otherwise None
|
||||
"""
|
||||
if depth is None or depth > 0:
|
||||
self.reduce_depth(depth)
|
||||
self.compute_report()
|
||||
try:
|
||||
if file_output:
|
||||
file_desc = open(file_output, 'w')
|
||||
else:
|
||||
file_desc = stdout
|
||||
except IOError as error:
|
||||
print("I/O error({0}): {1}".format(error.errno, error.strerror))
|
||||
return False
|
||||
|
||||
to_call = {'json': self.generate_json,
|
||||
'html': self.generate_html,
|
||||
'csv-ci': self.generate_csv,
|
||||
'table': self.generate_table}[export_format]
|
||||
output = to_call(file_desc)
|
||||
|
||||
if file_desc is not stdout:
|
||||
file_desc.close()
|
||||
|
||||
return output
|
||||
|
||||
@staticmethod
|
||||
def _move_up_tree(tree, next_module):
|
||||
tree.setdefault("children", [])
|
||||
for child in tree["children"]:
|
||||
if child["name"] == next_module:
|
||||
return child
|
||||
else:
|
||||
new_module = {"name": next_module, "value": 0, "delta": 0}
|
||||
tree["children"].append(new_module)
|
||||
return new_module
|
||||
|
||||
def generate_html(self, file_desc):
|
||||
"""Generate a json file from a memory map for D3
|
||||
|
||||
Positional arguments:
|
||||
file_desc - the file to write out the final report to
|
||||
"""
|
||||
tree_text = {"name": ".text", "value": 0, "delta": 0}
|
||||
tree_bss = {"name": ".bss", "value": 0, "delta": 0}
|
||||
tree_data = {"name": ".data", "value": 0, "delta": 0}
|
||||
for name, dct in self.modules.items():
|
||||
cur_text = tree_text
|
||||
cur_bss = tree_bss
|
||||
cur_data = tree_data
|
||||
modules = name.split(sep)
|
||||
while True:
|
||||
try:
|
||||
cur_text["value"] += dct['.text']
|
||||
cur_text["delta"] += dct['.text']
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
cur_bss["value"] += dct['.bss']
|
||||
cur_bss["delta"] += dct['.bss']
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
cur_data["value"] += dct['.data']
|
||||
cur_data["delta"] += dct['.data']
|
||||
except KeyError:
|
||||
pass
|
||||
if not modules:
|
||||
break
|
||||
next_module = modules.pop(0)
|
||||
cur_text = self._move_up_tree(cur_text, next_module)
|
||||
cur_data = self._move_up_tree(cur_data, next_module)
|
||||
cur_bss = self._move_up_tree(cur_bss, next_module)
|
||||
if self.old_modules:
|
||||
for name, dct in self.old_modules.items():
|
||||
cur_text = tree_text
|
||||
cur_bss = tree_bss
|
||||
cur_data = tree_data
|
||||
modules = name.split(sep)
|
||||
while True:
|
||||
try:
|
||||
cur_text["delta"] -= dct['.text']
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
cur_bss["delta"] -= dct['.bss']
|
||||
except KeyError:
|
||||
pass
|
||||
try:
|
||||
cur_data["delta"] -= dct['.data']
|
||||
except KeyError:
|
||||
pass
|
||||
if not modules:
|
||||
break
|
||||
next_module = modules.pop(0)
|
||||
if not any(
|
||||
cld['name'] == next_module
|
||||
for cld in cur_text['children']
|
||||
):
|
||||
break
|
||||
cur_text = self._move_up_tree(cur_text, next_module)
|
||||
cur_data = self._move_up_tree(cur_data, next_module)
|
||||
cur_bss = self._move_up_tree(cur_bss, next_module)
|
||||
|
||||
tree_rom = {
|
||||
"name": "ROM",
|
||||
"value": tree_text["value"] + tree_data["value"],
|
||||
"delta": tree_text["delta"] + tree_data["delta"],
|
||||
"children": [tree_text, tree_data]
|
||||
}
|
||||
tree_ram = {
|
||||
"name": "RAM",
|
||||
"value": tree_bss["value"] + tree_data["value"],
|
||||
"delta": tree_bss["delta"] + tree_data["delta"],
|
||||
"children": [tree_bss, tree_data]
|
||||
}
|
||||
|
||||
jinja_loader = FileSystemLoader(dirname(abspath(__file__)))
|
||||
jinja_environment = Environment(loader=jinja_loader,
|
||||
undefined=StrictUndefined)
|
||||
|
||||
template = jinja_environment.get_template("memap_flamegraph.html")
|
||||
name, _ = splitext(basename(file_desc.name))
|
||||
if name.endswith("_map"):
|
||||
name = name[:-4]
|
||||
if self.tc_name:
|
||||
name = "%s %s" % (name, self.tc_name)
|
||||
data = {
|
||||
"name": name,
|
||||
"rom": json.dumps(tree_rom),
|
||||
"ram": json.dumps(tree_ram),
|
||||
}
|
||||
file_desc.write(template.render(data))
|
||||
return None
|
||||
|
||||
def generate_json(self, file_desc):
|
||||
"""Generate a json file from a memory map
|
||||
|
||||
Positional arguments:
|
||||
file_desc - the file to write out the final report to
|
||||
"""
|
||||
file_desc.write(json.dumps(self.mem_report, indent=4))
|
||||
file_desc.write('\n')
|
||||
return None
|
||||
|
||||
RAM_FORMAT_STR = (
|
||||
"Total Static RAM memory (data + bss): {}({:+}) bytes\n"
|
||||
)
|
||||
|
||||
ROM_FORMAT_STR = (
|
||||
"Total Flash memory (text + data): {}({:+}) bytes\n"
|
||||
)
|
||||
|
||||
def generate_csv(self, file_desc):
|
||||
"""Generate a CSV file from a memoy map
|
||||
|
||||
Positional arguments:
|
||||
file_desc - the file to write out the final report to
|
||||
"""
|
||||
writer = csv.writer(file_desc, delimiter=',',
|
||||
quoting=csv.QUOTE_MINIMAL)
|
||||
|
||||
module_section = []
|
||||
sizes = []
|
||||
for i in sorted(self.short_modules):
|
||||
for k in self.print_sections + self.delta_sections:
|
||||
module_section.append((i + k))
|
||||
sizes += [self.short_modules[i][k]]
|
||||
|
||||
module_section.append('static_ram')
|
||||
sizes.append(self.mem_summary['static_ram'])
|
||||
|
||||
module_section.append('total_flash')
|
||||
sizes.append(self.mem_summary['total_flash'])
|
||||
|
||||
writer.writerow(module_section)
|
||||
writer.writerow(sizes)
|
||||
return None
|
||||
|
||||
def generate_table(self, file_desc):
|
||||
"""Generate a table from a memoy map
|
||||
|
||||
Returns: string of the generated table
|
||||
"""
|
||||
# Create table
|
||||
columns = ['Module']
|
||||
columns.extend(self.print_sections)
|
||||
|
||||
table = PrettyTable(columns, junction_char="|", hrules=HEADER)
|
||||
table.align["Module"] = "l"
|
||||
for col in self.print_sections:
|
||||
table.align[col] = 'r'
|
||||
|
||||
for i in list(self.print_sections):
|
||||
table.align[i] = 'r'
|
||||
|
||||
for i in sorted(self.short_modules):
|
||||
row = [i]
|
||||
|
||||
for k in self.print_sections:
|
||||
row.append("{}({:+})".format(
|
||||
self.short_modules[i][k],
|
||||
self.short_modules[i][k + "-delta"]
|
||||
))
|
||||
|
||||
table.add_row(row)
|
||||
|
||||
subtotal_row = ['Subtotals']
|
||||
for k in self.print_sections:
|
||||
subtotal_row.append("{}({:+})".format(
|
||||
self.subtotal[k], self.subtotal[k + '-delta']))
|
||||
|
||||
table.add_row(subtotal_row)
|
||||
|
||||
output = table.get_string()
|
||||
output += '\n'
|
||||
|
||||
output += self.RAM_FORMAT_STR.format(
|
||||
self.mem_summary['static_ram'],
|
||||
self.mem_summary['static_ram_delta']
|
||||
)
|
||||
output += self.ROM_FORMAT_STR.format(
|
||||
self.mem_summary['total_flash'],
|
||||
self.mem_summary['total_flash_delta']
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
toolchains = ["ARM", "ARM_STD", "ARM_MICRO", "GCC_ARM", "IAR"]
|
||||
|
||||
def compute_report(self):
|
||||
""" Generates summary of memory usage for main areas
|
||||
"""
|
||||
self.subtotal = defaultdict(int)
|
||||
|
||||
for mod in self.modules.values():
|
||||
for k in self.sections:
|
||||
self.subtotal[k] += mod[k]
|
||||
self.subtotal[k + '-delta'] += mod[k]
|
||||
if self.old_modules:
|
||||
for mod in self.old_modules.values():
|
||||
for k in self.sections:
|
||||
self.subtotal[k + '-delta'] -= mod[k]
|
||||
|
||||
self.mem_summary = {
|
||||
'static_ram': self.subtotal['.data'] + self.subtotal['.bss'],
|
||||
'static_ram_delta':
|
||||
self.subtotal['.data-delta'] + self.subtotal['.bss-delta'],
|
||||
'total_flash': (self.subtotal['.text'] + self.subtotal['.data']),
|
||||
'total_flash_delta':
|
||||
self.subtotal['.text-delta'] + self.subtotal['.data-delta'],
|
||||
}
|
||||
|
||||
self.mem_report = []
|
||||
if self.short_modules:
|
||||
for name, sizes in sorted(self.short_modules.items()):
|
||||
self.mem_report.append({
|
||||
"module": name,
|
||||
"size": {
|
||||
k: sizes.get(k, 0) for k in (self.print_sections +
|
||||
self.delta_sections)
|
||||
}
|
||||
})
|
||||
|
||||
self.mem_report.append({
|
||||
'summary': self.mem_summary
|
||||
})
|
||||
|
||||
def parse(self, mapfile, toolchain):
|
||||
""" Parse and decode map file depending on the toolchain
|
||||
|
||||
Positional arguments:
|
||||
mapfile - the file name of the memory map file
|
||||
toolchain - the toolchain used to create the file
|
||||
"""
|
||||
self.tc_name = toolchain.title()
|
||||
if toolchain in ("ARM", "ARM_STD", "ARM_MICRO", "ARMC6"):
|
||||
parser = _ArmccParser
|
||||
elif toolchain == "GCC_ARM":
|
||||
parser = _GccParser
|
||||
elif toolchain == "IAR":
|
||||
parser = _IarParser
|
||||
else:
|
||||
return False
|
||||
try:
|
||||
with open(mapfile, 'r') as file_input:
|
||||
self.modules = parser().parse_mapfile(file_input)
|
||||
try:
|
||||
with open("%s.old" % mapfile, 'r') as old_input:
|
||||
self.old_modules = parser().parse_mapfile(old_input)
|
||||
except IOError:
|
||||
self.old_modules = None
|
||||
return True
|
||||
|
||||
except IOError as error:
|
||||
print("I/O error({0}): {1}".format(error.errno, error.strerror))
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
"""Entry Point"""
|
||||
version = '0.4.0'
|
||||
|
||||
# Parser handling
|
||||
parser = ArgumentParser(
|
||||
description="Memory Map File Analyser for ARM mbed\nversion %s" %
|
||||
version)
|
||||
|
||||
parser.add_argument(
|
||||
'file', type=argparse_filestring_type, help='memory map file')
|
||||
|
||||
parser.add_argument(
|
||||
'-t', '--toolchain', dest='toolchain',
|
||||
help='select a toolchain used to build the memory map file (%s)' %
|
||||
", ".join(MemapParser.toolchains),
|
||||
required=True,
|
||||
type=argparse_uppercase_type(MemapParser.toolchains, "toolchain"))
|
||||
|
||||
parser.add_argument(
|
||||
'-d', '--depth', dest='depth', type=int,
|
||||
help='specify directory depth level to display report', required=False)
|
||||
|
||||
parser.add_argument(
|
||||
'-o', '--output', help='output file name', required=False)
|
||||
|
||||
parser.add_argument(
|
||||
'-e', '--export', dest='export', required=False, default='table',
|
||||
type=argparse_lowercase_hyphen_type(MemapParser.export_formats,
|
||||
'export format'),
|
||||
help="export format (examples: %s: default)" %
|
||||
", ".join(MemapParser.export_formats))
|
||||
|
||||
parser.add_argument('-v', '--version', action='version', version=version)
|
||||
|
||||
# Parse/run command
|
||||
if len(argv) <= 1:
|
||||
parser.print_help()
|
||||
exit(1)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Create memap object
|
||||
memap = MemapParser()
|
||||
|
||||
# Parse and decode a map file
|
||||
if args.file and args.toolchain:
|
||||
if memap.parse(args.file, args.toolchain) is False:
|
||||
exit(0)
|
||||
|
||||
if args.depth is None:
|
||||
depth = 2 # default depth level
|
||||
else:
|
||||
depth = args.depth
|
||||
|
||||
returned_string = None
|
||||
# Write output in file
|
||||
if args.output is not None:
|
||||
returned_string = memap.generate_output(
|
||||
args.export,
|
||||
depth,
|
||||
args.output
|
||||
)
|
||||
else: # Write output in screen
|
||||
returned_string = memap.generate_output(args.export, depth)
|
||||
|
||||
if args.export == 'table' and returned_string:
|
||||
print(returned_string)
|
||||
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,85 @@
|
||||
# memap - Static Memory Map Analysis
|
||||
|
||||
## Introduction
|
||||
|
||||
_memap_ is a simple utility that displays static memory information required by [mbed](https://github.com/mbedmicro/mbed) applications. This information is produced by analysing the memory map file previously generated by your toolchain.
|
||||
|
||||
**Note**: this tool shows static RAM usage and the total size of allocated heap and stack space defined at compile time, not the actual heap and stack usage (which may be different depending on your application).
|
||||
|
||||
## Table of contents
|
||||
|
||||
1. [Using memap](#using-memap)
|
||||
2. [Information on memory sections](#info-mem-sections)
|
||||
3. [Current support](#current-support)
|
||||
4. [Known problems](#known-problems)
|
||||
|
||||
## Using memap
|
||||
|
||||
_memap_ is automatically invoked after an mbed build finishes successfully. It’s also possible to manually run the program with different command line options, for example:
|
||||
|
||||
$> python memap.py
|
||||
usage: memap.py [-h] -t TOOLCHAIN [-o OUTPUT] [-e EXPORT] [-v] file
|
||||
|
||||
Memory Map File Analyser for ARM mbed version 0.3.11
|
||||
|
||||
positional arguments:
|
||||
file memory map file
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-t TOOLCHAIN, --toolchain TOOLCHAIN
|
||||
select a toolchain used to build the memory map file
|
||||
(ARM, GCC_ARM, IAR)
|
||||
-o OUTPUT, --output OUTPUT
|
||||
output file name
|
||||
-e EXPORT, --export EXPORT
|
||||
export format (examples: 'json', 'csv-ci', 'table':
|
||||
default)
|
||||
-v, --version show program's version number and exit
|
||||
|
||||
Result example:
|
||||
|
||||
$> python memap.py GCC_ARM\myprog3.map -t GCC_ARM
|
||||
|
||||
+----------------------------+-------+-------+------+
|
||||
| Module | .text | .data | .bss |
|
||||
+----------------------------+-------+-------+------+
|
||||
| Fill | 170 | 0 | 2294 |
|
||||
| Misc | 36282 | 2220 | 2152 |
|
||||
| core/hal | 15396 | 16 | 568 |
|
||||
| core/rtos | 6751 | 24 | 2662 |
|
||||
| features/FEATURE_IPV4 | 96 | 0 | 48 |
|
||||
| frameworks/greentea-client | 912 | 28 | 44 |
|
||||
| frameworks/utest | 3079 | 0 | 732 |
|
||||
| Subtotals | 62686 | 2288 | 8500 |
|
||||
+----------------------------+-------+-------+------+
|
||||
Allocated Heap: 65540 bytes
|
||||
Allocated Stack: 32768 bytes
|
||||
Total Static RAM memory (data + bss): 10788 bytes
|
||||
Total RAM memory (data + bss + heap + stack): 109096 bytes
|
||||
Total Flash memory (text + data + misc): 66014 bytes
|
||||
|
||||
## Information on memory sections
|
||||
|
||||
The table above showed multiple memory sections.
|
||||
|
||||
* `.text`: is where the code application and constants are located in Flash.
|
||||
* `.data`: non-zero initialized variables; allocated in both RAM and Flash memory (variables are copied from Flash to RAM at run time)
|
||||
* `.bss`: uninitialized data allocated in RAM, or variables initialized to zero.
|
||||
* `Heap`: dynamic allocations in the Heap area in RAM (for example, used by `malloc`). The maximum size value may be defined at build time.
|
||||
* `Stack`: dynamic allocations in the Stack area in RAM (for example, used to store local data, temporary data when branching to a subroutine or context switch information). The maximum size value may be defined at build time.
|
||||
|
||||
There are other entries that require a bit of clarification:
|
||||
|
||||
* Fill: represents the bytes in multiple sections (RAM and Flash) that the toolchain has filled with zeros because it requires subsequent data or code to be aligned appropriately in memory.
|
||||
* Misc: usually represents helper libraries introduced by the toolchain (like `libc`), but can also represent modules that are not part of mbed.
|
||||
|
||||
## Current support
|
||||
|
||||
_memap_ has been tested on Windows 7, Linux and Mac OS X and works with memory map files are generated by the GCC_ARM, ARM (ARM Compiler 5) and IAR toochains.
|
||||
|
||||
## Known issues and new features
|
||||
|
||||
This utility is considered ‘alpha’ quality at the moment. The information generated by this utility may not be fully accurate and may vary from one toolchain to another.
|
||||
|
||||
If you are experiencing problems, or would like additional features, please raise a ticket on [GitHub](https://github.com/mbedmicro/mbed/issues) and use `[memap]` in the title.
|
||||
@@ -0,0 +1,617 @@
|
||||
"""
|
||||
mbed SDK
|
||||
Copyright (c) 2011-2013 ARM Limited
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
from __future__ import print_function, division, absolute_import
|
||||
import sys
|
||||
import inspect
|
||||
import os
|
||||
import argparse
|
||||
import math
|
||||
from os import listdir, remove, makedirs
|
||||
from shutil import copyfile
|
||||
from os.path import isdir, join, exists, split, relpath, splitext, abspath
|
||||
from os.path import commonprefix, normpath, dirname
|
||||
from subprocess import Popen, PIPE, STDOUT, call
|
||||
from math import ceil
|
||||
import json
|
||||
from collections import OrderedDict
|
||||
import logging
|
||||
from intelhex import IntelHex
|
||||
import io
|
||||
|
||||
try:
|
||||
unicode
|
||||
except NameError:
|
||||
unicode = str
|
||||
|
||||
def remove_if_in(lst, thing):
|
||||
if thing in lst:
|
||||
lst.remove(thing)
|
||||
|
||||
def compile_worker(job):
|
||||
"""Standard task runner used for compiling
|
||||
|
||||
Positional argumets:
|
||||
job - a dict containing a list of commands and the remaining arguments
|
||||
to run_cmd
|
||||
"""
|
||||
results = []
|
||||
for command in job['commands']:
|
||||
try:
|
||||
_, _stderr, _rc = run_cmd(command, work_dir=job['work_dir'],
|
||||
chroot=job['chroot'])
|
||||
except KeyboardInterrupt:
|
||||
raise ToolException
|
||||
|
||||
results.append({
|
||||
'code': _rc,
|
||||
'output': _stderr,
|
||||
'command': command
|
||||
})
|
||||
|
||||
return {
|
||||
'source': job['source'],
|
||||
'object': job['object'],
|
||||
'commands': job['commands'],
|
||||
'results': results
|
||||
}
|
||||
|
||||
def cmd(command, check=True, verbose=False, shell=False, cwd=None):
|
||||
"""A wrapper to run a command as a blocking job"""
|
||||
text = command if shell else ' '.join(command)
|
||||
if verbose:
|
||||
print(text)
|
||||
return_code = call(command, shell=shell, cwd=cwd)
|
||||
if check and return_code != 0:
|
||||
raise Exception('ERROR %d: "%s"' % (return_code, text))
|
||||
|
||||
|
||||
def run_cmd(command, work_dir=None, chroot=None, redirect=False):
|
||||
"""Run a command in the foreground
|
||||
|
||||
Positional arguments:
|
||||
command - the command to run
|
||||
|
||||
Keyword arguments:
|
||||
work_dir - the working directory to run the command in
|
||||
chroot - the chroot to run the command in
|
||||
redirect - redirect the stderr to a pipe to be used later
|
||||
"""
|
||||
if chroot:
|
||||
# Conventions managed by the web team for the mbed.org build system
|
||||
chroot_cmd = [
|
||||
'/usr/sbin/chroot', '--userspec=33:33', chroot
|
||||
]
|
||||
for element in command:
|
||||
chroot_cmd += [element.replace(chroot, '')]
|
||||
|
||||
logging.debug("Running command %s", ' '.join(chroot_cmd))
|
||||
command = chroot_cmd
|
||||
work_dir = None
|
||||
|
||||
try:
|
||||
process = Popen(command, stdout=PIPE,
|
||||
stderr=STDOUT if redirect else PIPE, cwd=work_dir,
|
||||
universal_newlines=True)
|
||||
_stdout, _stderr = process.communicate()
|
||||
except OSError:
|
||||
print("[OS ERROR] Command: "+(' '.join(command)))
|
||||
raise
|
||||
|
||||
return _stdout, _stderr, process.returncode
|
||||
|
||||
|
||||
def run_cmd_ext(command):
|
||||
""" A version of run command that checks if the command exists befor running
|
||||
|
||||
Positional arguments:
|
||||
command - the command line you are trying to invoke
|
||||
"""
|
||||
assert is_cmd_valid(command[0])
|
||||
process = Popen(command, stdout=PIPE, stderr=PIPE)
|
||||
_stdout, _stderr = process.communicate()
|
||||
return _stdout, _stderr, process.returncode
|
||||
|
||||
|
||||
def is_cmd_valid(command):
|
||||
""" Verify that a command exists and is executable
|
||||
|
||||
Positional arguments:
|
||||
command - the command to check
|
||||
"""
|
||||
caller = get_caller_name()
|
||||
cmd_path = find_cmd_abspath(command)
|
||||
if not cmd_path:
|
||||
error("%s: Command '%s' can't be found" % (caller, command))
|
||||
if not is_exec(cmd_path):
|
||||
error("%s: Command '%s' resolves to file '%s' which is not executable"
|
||||
% (caller, command, cmd_path))
|
||||
return True
|
||||
|
||||
|
||||
def is_exec(path):
|
||||
"""A simple check to verify that a path to an executable exists
|
||||
|
||||
Positional arguments:
|
||||
path - the executable
|
||||
"""
|
||||
return os.access(path, os.X_OK) or os.access(path+'.exe', os.X_OK)
|
||||
|
||||
|
||||
def find_cmd_abspath(command):
|
||||
""" Returns the absolute path to a command.
|
||||
None is returned if no absolute path was found.
|
||||
|
||||
Positional arguhments:
|
||||
command - the command to find the path of
|
||||
"""
|
||||
if exists(command) or exists(command + '.exe'):
|
||||
return os.path.abspath(command)
|
||||
if not 'PATH' in os.environ:
|
||||
raise Exception("Can't find command path for current platform ('%s')"
|
||||
% sys.platform)
|
||||
path_env = os.environ['PATH']
|
||||
for path in path_env.split(os.pathsep):
|
||||
cmd_path = '%s/%s' % (path, command)
|
||||
if exists(cmd_path) or exists(cmd_path + '.exe'):
|
||||
return cmd_path
|
||||
|
||||
|
||||
def mkdir(path):
|
||||
""" a wrapped makedirs that only tries to create a directory if it does not
|
||||
exist already
|
||||
|
||||
Positional arguments:
|
||||
path - the path to maybe create
|
||||
"""
|
||||
if not exists(path):
|
||||
makedirs(path)
|
||||
|
||||
|
||||
def write_json_to_file(json_data, file_name):
|
||||
"""
|
||||
Write json content in file
|
||||
:param json_data:
|
||||
:param file_name:
|
||||
:return:
|
||||
"""
|
||||
# Create the target dir for file if necessary
|
||||
test_spec_dir = os.path.dirname(file_name)
|
||||
|
||||
if test_spec_dir:
|
||||
mkdir(test_spec_dir)
|
||||
|
||||
try:
|
||||
with open(file_name, 'w') as f:
|
||||
f.write(json.dumps(json_data, indent=2))
|
||||
except IOError as e:
|
||||
print("[ERROR] Error writing test spec to file")
|
||||
print(e)
|
||||
|
||||
|
||||
def copy_file(src, dst):
|
||||
""" Implement the behaviour of "shutil.copy(src, dst)" without copying the
|
||||
permissions (this was causing errors with directories mounted with samba)
|
||||
|
||||
Positional arguments:
|
||||
src - the source of the copy operation
|
||||
dst - the destination of the copy operation
|
||||
"""
|
||||
if isdir(dst):
|
||||
_, base = split(src)
|
||||
dst = join(dst, base)
|
||||
copyfile(src, dst)
|
||||
|
||||
|
||||
def copy_when_different(src, dst):
|
||||
""" Only copy the file when it's different from its destination.
|
||||
|
||||
Positional arguments:
|
||||
src - the source of the copy operation
|
||||
dst - the destination of the copy operation
|
||||
"""
|
||||
if isdir(dst):
|
||||
_, base = split(src)
|
||||
dst = join(dst, base)
|
||||
if exists(dst):
|
||||
with open(src, 'rb') as srcfd, open(dst, 'rb') as dstfd:
|
||||
if srcfd.read() == dstfd.read():
|
||||
return
|
||||
copyfile(src, dst)
|
||||
|
||||
|
||||
def delete_dir_files(directory):
|
||||
""" A function that does rm -rf
|
||||
|
||||
Positional arguments:
|
||||
directory - the directory to remove
|
||||
"""
|
||||
if not exists(directory):
|
||||
return
|
||||
|
||||
for element in listdir(directory):
|
||||
to_remove = join(directory, element)
|
||||
if not isdir(to_remove):
|
||||
remove(to_remove)
|
||||
|
||||
|
||||
def get_caller_name(steps=2):
|
||||
"""
|
||||
When called inside a function, it returns the name
|
||||
of the caller of that function.
|
||||
|
||||
Keyword arguments:
|
||||
steps - the number of steps up the stack the calling function is
|
||||
"""
|
||||
return inspect.stack()[steps][3]
|
||||
|
||||
|
||||
def error(msg):
|
||||
"""Fatal error, abort hard
|
||||
|
||||
Positional arguments:
|
||||
msg - the message to print before crashing
|
||||
"""
|
||||
print("ERROR: %s" % msg)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def rel_path(path, base, dot=False):
|
||||
"""Relative path calculation that optionaly always starts with a dot
|
||||
|
||||
Positional arguments:
|
||||
path - the path to make relative
|
||||
base - what to make the path relative to
|
||||
|
||||
Keyword arguments:
|
||||
dot - if True, the path will always start with a './'
|
||||
"""
|
||||
final_path = relpath(path, base)
|
||||
if dot and not final_path.startswith('.'):
|
||||
final_path = './' + final_path
|
||||
return final_path
|
||||
|
||||
|
||||
class ToolException(Exception):
|
||||
"""A class representing an exception throw by the tools"""
|
||||
pass
|
||||
|
||||
class NotSupportedException(Exception):
|
||||
"""A class a toolchain not supporting a particular target"""
|
||||
pass
|
||||
|
||||
class InvalidReleaseTargetException(Exception):
|
||||
pass
|
||||
|
||||
class NoValidToolchainException(Exception):
|
||||
"""A class representing no valid toolchain configurations found on
|
||||
the system"""
|
||||
pass
|
||||
|
||||
def split_path(path):
|
||||
"""spilt a file name into it's directory name, base name, and extension
|
||||
|
||||
Positional arguments:
|
||||
path - the file name to split
|
||||
"""
|
||||
base, has_ext = split(path)
|
||||
name, ext = splitext(has_ext)
|
||||
return base, name, ext
|
||||
|
||||
|
||||
def get_path_depth(path):
|
||||
""" Given a path, return the number of directory levels present.
|
||||
This roughly translates to the number of path separators (os.sep) + 1.
|
||||
Ex. Given "path/to/dir", this would return 3
|
||||
Special cases: "." and "/" return 0
|
||||
|
||||
Positional arguments:
|
||||
path - the path to calculate the depth of
|
||||
"""
|
||||
normalized_path = normpath(path)
|
||||
path_depth = 0
|
||||
head, tail = split(normalized_path)
|
||||
|
||||
while tail and tail != '.':
|
||||
path_depth += 1
|
||||
head, tail = split(head)
|
||||
|
||||
return path_depth
|
||||
|
||||
|
||||
def args_error(parser, message):
|
||||
"""Abort with an error that was generated by the arguments to a CLI program
|
||||
|
||||
Positional arguments:
|
||||
parser - the ArgumentParser object that parsed the command line
|
||||
message - what went wrong
|
||||
"""
|
||||
parser.exit(status=2, message=message+'\n')
|
||||
|
||||
|
||||
def construct_enum(**enums):
|
||||
""" Create your own pseudo-enums
|
||||
|
||||
Keyword arguments:
|
||||
* - a member of the Enum you are creating and it's value
|
||||
"""
|
||||
return type('Enum', (), enums)
|
||||
|
||||
|
||||
def check_required_modules(required_modules, verbose=True):
|
||||
""" Function checks for Python modules which should be "importable"
|
||||
before test suite can be used.
|
||||
@return returns True if all modules are installed already
|
||||
"""
|
||||
import imp
|
||||
not_installed_modules = []
|
||||
for module_name in required_modules:
|
||||
try:
|
||||
imp.find_module(module_name)
|
||||
except ImportError:
|
||||
# We also test against a rare case: module is an egg file
|
||||
try:
|
||||
__import__(module_name)
|
||||
except ImportError as exc:
|
||||
not_installed_modules.append(module_name)
|
||||
if verbose:
|
||||
print("Error: %s" % exc)
|
||||
|
||||
if verbose:
|
||||
if not_installed_modules:
|
||||
print("Warning: Module(s) %s not installed. Please install "
|
||||
"required module(s) before using this script."
|
||||
% (', '.join(not_installed_modules)))
|
||||
|
||||
if not_installed_modules:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def _ordered_dict_collapse_dups(pair_list):
|
||||
to_ret = OrderedDict()
|
||||
for key, value in pair_list:
|
||||
if key in to_ret:
|
||||
if isinstance(to_ret[key], dict):
|
||||
to_ret[key].update(value)
|
||||
elif isinstance(to_ret[key], list):
|
||||
to_ret[key].extend(value)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Key %s found twice and is not mergeable" % key
|
||||
)
|
||||
else:
|
||||
to_ret[key] = value
|
||||
return to_ret
|
||||
|
||||
|
||||
def json_file_to_dict(fname):
|
||||
""" Read a JSON file and return its Python representation, transforming all
|
||||
the strings from Unicode to ASCII. The order of keys in the JSON file is
|
||||
preserved.
|
||||
|
||||
Positional arguments:
|
||||
fname - the name of the file to parse
|
||||
"""
|
||||
try:
|
||||
with io.open(fname, encoding='ascii',
|
||||
errors='ignore') as file_obj:
|
||||
return json.load(
|
||||
file_obj, object_pairs_hook=_ordered_dict_collapse_dups
|
||||
)
|
||||
except (ValueError, IOError) as e:
|
||||
sys.stderr.write("Error parsing '%s': %s\n" % (fname, e))
|
||||
raise
|
||||
|
||||
# Wowza, double closure
|
||||
def argparse_type(casedness, prefer_hyphen=False):
|
||||
def middle(lst, type_name):
|
||||
def parse_type(string):
|
||||
""" validate that an argument passed in (as string) is a member of
|
||||
the list of possible arguments. Offer a suggestion if the case of
|
||||
the string, or the hyphens/underscores do not match the expected
|
||||
style of the argument.
|
||||
"""
|
||||
if not isinstance(string, unicode):
|
||||
string = string.decode()
|
||||
if prefer_hyphen:
|
||||
newstring = casedness(string).replace("_", "-")
|
||||
else:
|
||||
newstring = casedness(string).replace("-", "_")
|
||||
if string in lst:
|
||||
return string
|
||||
elif string not in lst and newstring in lst:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"{0} is not a supported {1}. Did you mean {2}?".format(
|
||||
string, type_name, newstring))
|
||||
else:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"{0} is not a supported {1}. Supported {1}s are:\n{2}".
|
||||
format(string, type_name, columnate(lst)))
|
||||
return parse_type
|
||||
return middle
|
||||
|
||||
# short cuts for the argparse_type versions
|
||||
argparse_uppercase_type = argparse_type(unicode.upper, False)
|
||||
argparse_lowercase_type = argparse_type(unicode.lower, False)
|
||||
argparse_uppercase_hyphen_type = argparse_type(unicode.upper, True)
|
||||
argparse_lowercase_hyphen_type = argparse_type(unicode.lower, True)
|
||||
|
||||
def argparse_force_type(case):
|
||||
""" validate that an argument passed in (as string) is a member of the list
|
||||
of possible arguments after converting it's case.
|
||||
"""
|
||||
def middle(lst, type_name):
|
||||
""" The parser type generator"""
|
||||
if not isinstance(lst[0], unicode):
|
||||
lst = [o.decode() for o in lst]
|
||||
def parse_type(string):
|
||||
""" The parser type"""
|
||||
if not isinstance(string, unicode):
|
||||
string = string.decode()
|
||||
for option in lst:
|
||||
if case(string) == case(option):
|
||||
return option
|
||||
raise argparse.ArgumentTypeError(
|
||||
"{0} is not a supported {1}. Supported {1}s are:\n{2}".
|
||||
format(string, type_name, columnate(lst)))
|
||||
return parse_type
|
||||
return middle
|
||||
|
||||
# these two types convert the case of their arguments _before_ validation
|
||||
argparse_force_uppercase_type = argparse_force_type(unicode.upper)
|
||||
argparse_force_lowercase_type = argparse_force_type(unicode.lower)
|
||||
|
||||
def argparse_many(func):
|
||||
""" An argument parser combinator that takes in an argument parser and
|
||||
creates a new parser that accepts a comma separated list of the same thing.
|
||||
"""
|
||||
def wrap(string):
|
||||
""" The actual parser"""
|
||||
return [func(s) for s in string.split(",")]
|
||||
return wrap
|
||||
|
||||
def argparse_filestring_type(string):
|
||||
""" An argument parser that verifies that a string passed in corresponds
|
||||
to a file"""
|
||||
if exists(string):
|
||||
return string
|
||||
else:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"{0}"" does not exist in the filesystem.".format(string))
|
||||
|
||||
def argparse_profile_filestring_type(string):
|
||||
""" An argument parser that verifies that a string passed in is either
|
||||
absolute path or a file name (expanded to
|
||||
mbed-os/tools/profiles/<fname>.json) of a existing file"""
|
||||
fpath = join(dirname(__file__), "profiles/{}.json".format(string))
|
||||
|
||||
# default profiles are searched first, local ones next.
|
||||
if exists(fpath):
|
||||
return fpath
|
||||
elif exists(string):
|
||||
return string
|
||||
else:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"{0} does not exist in the filesystem.".format(string))
|
||||
|
||||
def columnate(strings, separator=", ", chars=80):
|
||||
""" render a list of strings as a in a bunch of columns
|
||||
|
||||
Positional arguments:
|
||||
strings - the strings to columnate
|
||||
|
||||
Keyword arguments;
|
||||
separator - the separation between the columns
|
||||
chars - the maximum with of a row
|
||||
"""
|
||||
col_width = max(len(s) for s in strings)
|
||||
total_width = col_width + len(separator)
|
||||
columns = math.floor(chars / total_width)
|
||||
output = ""
|
||||
for i, string in zip(range(len(strings)), strings):
|
||||
append = string
|
||||
if i != len(strings) - 1:
|
||||
append += separator
|
||||
if i % columns == columns - 1:
|
||||
append += "\n"
|
||||
else:
|
||||
append = append.ljust(total_width)
|
||||
output += append
|
||||
return output
|
||||
|
||||
def argparse_dir_not_parent(other):
|
||||
"""fail if argument provided is a parent of the specified directory"""
|
||||
def parse_type(not_parent):
|
||||
"""The parser type"""
|
||||
abs_other = abspath(other)
|
||||
abs_not_parent = abspath(not_parent)
|
||||
if abs_not_parent == commonprefix([abs_not_parent, abs_other]):
|
||||
raise argparse.ArgumentTypeError(
|
||||
"{0} may not be a parent directory of {1}".format(
|
||||
not_parent, other))
|
||||
else:
|
||||
return not_parent
|
||||
return parse_type
|
||||
|
||||
def argparse_deprecate(replacement_message):
|
||||
"""fail if argument is provided with deprecation warning"""
|
||||
def parse_type(_):
|
||||
"""The parser type"""
|
||||
raise argparse.ArgumentTypeError("Deprecated." + replacement_message)
|
||||
return parse_type
|
||||
|
||||
def print_large_string(large_string):
|
||||
""" Breaks a string up into smaller pieces before print them
|
||||
|
||||
This is a limitation within Windows, as detailed here:
|
||||
https://bugs.python.org/issue11395
|
||||
|
||||
Positional arguments:
|
||||
large_string - the large string to print
|
||||
"""
|
||||
string_limit = 1000
|
||||
large_string_len = len(large_string)
|
||||
num_parts = int(ceil(float(large_string_len) / float(string_limit)))
|
||||
for string_part in range(num_parts):
|
||||
start_index = string_part * string_limit
|
||||
if string_part == num_parts - 1:
|
||||
sys.stdout.write(large_string[start_index:])
|
||||
else:
|
||||
sys.stdout.write(large_string[start_index:
|
||||
start_index + string_limit])
|
||||
sys.stdout.write("\n")
|
||||
|
||||
def intelhex_offset(filename, offset):
|
||||
"""Load a hex or bin file at a particular offset"""
|
||||
_, inteltype = splitext(filename)
|
||||
ih = IntelHex()
|
||||
if inteltype == ".bin":
|
||||
ih.loadbin(filename, offset=offset)
|
||||
elif inteltype == ".hex":
|
||||
ih.loadhex(filename)
|
||||
else:
|
||||
raise ToolException("File %s does not have a known binary file type"
|
||||
% filename)
|
||||
return ih
|
||||
|
||||
def integer(maybe_string, base):
|
||||
"""Make an integer of a number or a string"""
|
||||
if isinstance(maybe_string, int):
|
||||
return maybe_string
|
||||
else:
|
||||
return int(maybe_string, base)
|
||||
|
||||
def generate_update_filename(name, target):
|
||||
return "%s_update.%s" % (
|
||||
name,
|
||||
getattr(target, "OUTPUT_EXT_UPDATE", "bin")
|
||||
)
|
||||
|
||||
def print_end_warnings(end_warnings):
|
||||
""" Print a formatted list of warnings
|
||||
|
||||
Positional arguments:
|
||||
end_warnings - A list of warnings (strings) to print
|
||||
"""
|
||||
if end_warnings:
|
||||
warning_separator = "-" * 60
|
||||
print(warning_separator)
|
||||
for end_warning in end_warnings:
|
||||
print(end_warning)
|
||||
print(warning_separator)
|
||||
Reference in New Issue
Block a user