mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-10-28 00:35:47 +00:00
Merge pull request #9572 from JJenkx/windows/symbolize-sampler-stacks
feat(tools): symbolize the Windows samplers' stacks
This commit is contained in:
5 files changed
+521
-7
No files matched your search
@@ -252,6 +252,8 @@ if(BUILD_TESTING)
|
||||
endforeach()
|
||||
add_test(NAME trace_digest COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_trace_digest.py)
|
||||
add_test(NAME symbolize_samples COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_symbolize_samples.py)
|
||||
add_test(NAME conventions_bom_shebang COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_bom_shebang.py)
|
||||
set_tests_properties(conventions_bom_shebang PROPERTIES TIMEOUT 20)
|
||||
|
||||
@@ -163,18 +163,29 @@ struct Sampler {
|
||||
return frame;
|
||||
}
|
||||
|
||||
static void writeModules(std::FILE* file, const std::map<std::uint64_t, std::string>& loaded) {
|
||||
for (const auto& [base, name] : loaded) {
|
||||
const auto* dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
|
||||
const auto* nt = reinterpret_cast<const IMAGE_NT_HEADERS*>(base + dos->e_lfanew);
|
||||
std::fprintf(file, "# module %s 0x%lx 0x%lx\n", name.c_str(), static_cast<unsigned long>(nt->OptionalHeader.SizeOfImage), static_cast<unsigned long>(nt->FileHeader.TimeDateStamp));
|
||||
}
|
||||
}
|
||||
|
||||
void write() {
|
||||
std::lock_guard lock(mutex);
|
||||
std::FILE* file = std::fopen(path.c_str(), "w");
|
||||
if (file == nullptr) return;
|
||||
std::fprintf(file, "# %llu samples\n", static_cast<unsigned long long>(samples));
|
||||
std::map<std::uint64_t, std::string> loaded;
|
||||
for (const auto& [address, count] : counts) {
|
||||
HMODULE module = nullptr;
|
||||
char name[MAX_PATH] = "?";
|
||||
if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast<LPCSTR>(address), &module)) GetModuleFileNameA(module, name, sizeof(name));
|
||||
const char* base = std::strrchr(name, '\\');
|
||||
if (module != nullptr) loaded.emplace(reinterpret_cast<std::uint64_t>(module), base ? base + 1 : name);
|
||||
std::fprintf(file, "%s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast<unsigned long long>(address - reinterpret_cast<std::uint64_t>(module)), static_cast<unsigned long long>(count.first), static_cast<unsigned long long>(count.second));
|
||||
}
|
||||
writeModules(file, loaded);
|
||||
std::fclose(file);
|
||||
if (keepStacks) writeStacks();
|
||||
}
|
||||
@@ -195,11 +206,7 @@ struct Sampler {
|
||||
const double tickUs = 1e6 / static_cast<double>(frequency.QuadPart);
|
||||
std::fprintf(file, "# samples=%llu depth=%d truncated=%llu suspended_us_mean=%.1f suspended_us_max=%.1f\n", static_cast<unsigned long long>(samples), depth, static_cast<unsigned long long>(stacks.Truncated()),
|
||||
samples != 0 ? static_cast<double>(suspendedTicks) * tickUs / static_cast<double>(samples) : 0.0, static_cast<double>(longestSuspendedTicks) * tickUs);
|
||||
for (const auto& [base, name] : loaded) {
|
||||
const auto* dos = reinterpret_cast<const IMAGE_DOS_HEADER*>(base);
|
||||
const auto* nt = reinterpret_cast<const IMAGE_NT_HEADERS*>(base + dos->e_lfanew);
|
||||
std::fprintf(file, "# module %s 0x%lx 0x%lx\n", name.c_str(), static_cast<unsigned long>(nt->OptionalHeader.SizeOfImage), static_cast<unsigned long>(nt->FileHeader.TimeDateStamp));
|
||||
}
|
||||
writeModules(file, loaded);
|
||||
const auto text = stacks.Format([&](std::uint64_t address) { return names.at(address); });
|
||||
const bool written = std::fwrite(text.data(), 1, text.size(), file) == text.size();
|
||||
if (std::fclose(file) != 0 || !written) return;
|
||||
@@ -246,6 +253,7 @@ struct ProcessSampler {
|
||||
std::FILE* file = std::fopen(path.c_str(), "w");
|
||||
if (file == nullptr) return;
|
||||
std::fprintf(file, "# %llu rounds\n", static_cast<unsigned long long>(rounds));
|
||||
std::map<std::uint64_t, std::string> loaded;
|
||||
for (auto& [id, thread] : threads) {
|
||||
std::lock_guard lock(thread.mutex);
|
||||
std::vector<std::pair<std::uint64_t, std::pair<std::uint64_t, std::uint64_t>>> hot(thread.counts.begin(), thread.counts.end());
|
||||
@@ -260,9 +268,11 @@ struct ProcessSampler {
|
||||
char name[MAX_PATH] = "?";
|
||||
if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast<LPCSTR>(address), &module)) GetModuleFileNameA(module, name, sizeof(name));
|
||||
const char* base = std::strrchr(name, '\\');
|
||||
if (module != nullptr) loaded.emplace(reinterpret_cast<std::uint64_t>(module), base ? base + 1 : name);
|
||||
std::fprintf(file, " %s 0x%llx %llu %llu\n", base ? base + 1 : name, static_cast<unsigned long long>(address - reinterpret_cast<std::uint64_t>(module)), static_cast<unsigned long long>(hot[i].second.first), static_cast<unsigned long long>(hot[i].second.second));
|
||||
}
|
||||
}
|
||||
Sampler::writeModules(file, loaded);
|
||||
std::fclose(file);
|
||||
}
|
||||
};
|
||||
|
||||
+2
-2
@@ -216,11 +216,11 @@ On a device that supports `VK_EXT_device_fault`, the first `VK_ERROR_DEVICE_LOST
|
||||
|
||||
## Worker sampler
|
||||
|
||||
Set `APS5_SAMPLE_WORKER=<file>` to sample queue worker 0 every millisecond (in practice every timer tick). Every 20 s and at exit it writes `<file>`, one `module offset self inclusive` line per sampled address, and `<file stem>.collapsed.txt`, the worker's whole stacks in the collapsed format that speedscope.app and flamegraph.pl read: one `frame;...;frame count` line per distinct stack, root first, each frame as `module+0xRVA`. A walk that ends before the thread's root starts with `[truncated]`. The `#` lines give the samples, the walk depth (64 frames), the truncated walks, the mean and longest time the worker was suspended, and the size and PE timestamp of each module, to check the binary a file is symbolized against. A frame with `.pdata` is stepped by `RtlVirtualUnwind`; any other frame by its DWARF record in `.ehfram`, the unwind data the libraries are built with. Those steps read only inside the worker's stack, and a module is indexed after the worker resumes, the first time a walk meets it.
|
||||
Set `APS5_SAMPLE_WORKER=<file>` to sample queue worker 0 every millisecond (in practice every timer tick). Every 20 s and at exit it writes `<file>`, one `module offset self inclusive` line per sampled address, and `<file stem>.collapsed.txt`, the worker's whole stacks in the collapsed format that speedscope.app and flamegraph.pl read: one `frame;...;frame count` line per distinct stack, root first, each frame as `module+0xRVA`. A walk that ends before the thread's root starts with `[truncated]`. The `#` lines give the samples, the walk depth (64 frames), the truncated walks, the mean and longest time the worker was suspended, and the size and PE timestamp of each module, to check the binary a file is symbolized against. A frame with `.pdata` is stepped by `RtlVirtualUnwind`; any other frame by its DWARF record in `.ehfram`, the unwind data the libraries are built with. Those steps read only inside the worker's stack, and a module is indexed after the worker resumes, the first time a walk meets it. `python tools/symbolize_samples.py <file stem>.collapsed.txt --libs <libs directory>` names each frame from the COFF symbols of the module that ran, or from its export table when it has none, and leaves a module unsymbolized when its size or timestamp differs from its `# module` line or when it has none (`--unverified` symbolizes it anyway); `--since <earlier copy>` keeps only the samples taken after that copy, `--out` writes the named stacks and `--demangle` uses `c++filt`. Given `<file>`, it lists functions by self and by inclusive samples, where a function's inclusive count is the sum of its addresses' and overcounts a function that is on one stack more than once.
|
||||
|
||||
`APS5_SAMPLE_COMMITTER=<file>` samples queue 0's draw committer the same way and writes the same two files. The committer records the draws the worker hands over (`APS5_PIPELINED_DRAWS`), so when the worker's stacks wait in `DrawPipeline::Enqueue`, its stacks show what bounds the frame. It starts with the first pipelined draw; with `APS5_PIPELINED_DRAWS=0` nothing is sampled.
|
||||
|
||||
`APS5_SAMPLE_THREADS=<file>` samples every thread of the process every 2 ms and writes the 40 hottest addresses of each busy thread to `<file>`, 12 frames deep and through `.pdata` only. The samplers exist only on Windows; the addresses are symbolized offline, for example with `nm` on the `.prx`.
|
||||
`APS5_SAMPLE_THREADS=<file>` samples every thread of the process every 2 ms and writes the 40 hottest addresses of each busy thread to `<file>`, 12 frames deep and through `.pdata` only, under a `thread <id> samples <n>` line per thread and with the same `# module` lines. The samplers exist only on Windows; the addresses are symbolized offline with `tools/symbolize_samples.py`, which adds up all threads or, with `--thread <id>`, keeps one.
|
||||
|
||||
## Performance trace
|
||||
|
||||
|
||||
@@ -0,0 +1,300 @@
|
||||
import argparse
|
||||
import bisect
|
||||
import collections
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
DESCRIPTION = """Symbolize the output of the Windows samplers (APS5_SAMPLE_WORKER, APS5_SAMPLE_COMMITTER and
|
||||
APS5_SAMPLE_THREADS) against the modules that ran. Each sampled frame is written as module+0xRVA; this reads the
|
||||
COFF symbol table of each module found in --libs (the MinGW-built .prx and .dll files keep one), or its export
|
||||
table when it has none, and names every frame module!function.
|
||||
|
||||
The '# module name size timestamp' header lines the samplers write are compared with the SizeOfImage and
|
||||
TimeDateStamp of the file that is found; a module whose file does not match is left unsymbolized, since its
|
||||
addresses would name the wrong functions.
|
||||
|
||||
The sampler rewrites its files every 20 s with totals since start-up. --since takes an earlier copy of the same
|
||||
file and subtracts it, which keeps only the samples taken between the two copies (a steady-state window).
|
||||
|
||||
A module with no such record is also left unsymbolized, since nothing shows which build it was; --unverified
|
||||
symbolizes it anyway against the file found.
|
||||
|
||||
Input: a collapsed-stack file (<file stem>.collapsed.txt, 'frame;...;frame count' lines) or an address file
|
||||
('module offset self inclusive' lines, under 'thread <id> samples <n>' headings for APS5_SAMPLE_THREADS; --thread
|
||||
keeps one thread). Output: the hottest functions by self and by inclusive samples, and with --out a symbolized
|
||||
collapsed-stack file for speedscope.app or flamegraph.pl. For an address file, a function's inclusive count is the
|
||||
sum of its addresses' counts, which overcounts the samples in which it is on the stack more than once."""
|
||||
|
||||
IMAGE_SCN_CNT_CODE = 0x20
|
||||
IMAGE_SYM_CLASS_EXTERNAL = 2
|
||||
IMAGE_SYM_CLASS_STATIC = 3
|
||||
|
||||
|
||||
class PeModule:
|
||||
def __init__(self, path):
|
||||
data = Path(path).read_bytes()
|
||||
if data[:2] != b"MZ":
|
||||
raise ValueError(f"{path}: not a PE file")
|
||||
pe = struct.unpack_from("<I", data, 0x3C)[0]
|
||||
if data[pe:pe + 4] != b"PE\0\0":
|
||||
raise ValueError(f"{path}: not a PE file")
|
||||
_, sections, self.timestamp, symbol_table, symbol_count, optional_size, _ = struct.unpack_from("<HHIIIHH", data, pe + 4)
|
||||
optional = pe + 24
|
||||
if struct.unpack_from("<H", data, optional)[0] != 0x20B:
|
||||
raise ValueError(f"{path}: not a PE32+ image")
|
||||
self.image_base = struct.unpack_from("<Q", data, optional + 24)[0]
|
||||
self.size = struct.unpack_from("<I", data, optional + 56)[0]
|
||||
export_rva, export_size = struct.unpack_from("<II", data, optional + 112)
|
||||
self.sections = []
|
||||
for index in range(sections):
|
||||
entry = optional + optional_size + index * 40
|
||||
virtual_size, rva, raw_size, raw_pointer = struct.unpack_from("<IIII", data, entry + 8)
|
||||
characteristics = struct.unpack_from("<I", data, entry + 36)[0]
|
||||
self.sections.append((rva, max(virtual_size, raw_size), raw_pointer, characteristics))
|
||||
symbols = self.coff_symbols(data, symbol_table, symbol_count) if symbol_table and symbol_count else []
|
||||
self.exported = not symbols
|
||||
if self.exported and export_rva:
|
||||
symbols = self.export_symbols(data, export_rva, export_size)
|
||||
symbols.sort()
|
||||
self.code = []
|
||||
for start, size, _, characteristics in self.sections:
|
||||
if not characteristics & IMAGE_SCN_CNT_CODE:
|
||||
continue
|
||||
inside = [(address, name) for address, name in symbols if start <= address < start + size]
|
||||
self.code.append((start, start + size, [address for address, _ in inside], [name for _, name in inside]))
|
||||
self.names = [name for _, name in symbols]
|
||||
|
||||
def offset(self, rva):
|
||||
for start, size, raw, _ in self.sections:
|
||||
if start <= rva < start + size:
|
||||
return raw + rva - start
|
||||
raise ValueError(f"RVA {rva:#x} is in no section")
|
||||
|
||||
def coff_symbols(self, data, table, count):
|
||||
strings = table + count * 18
|
||||
result = []
|
||||
index = 0
|
||||
while index < count:
|
||||
entry = table + index * 18
|
||||
name, value, section, kind, storage, aux = struct.unpack_from("<8sIhHBB", data, entry)
|
||||
index += 1 + aux
|
||||
if storage not in (IMAGE_SYM_CLASS_EXTERNAL, IMAGE_SYM_CLASS_STATIC) or not 1 <= section <= len(self.sections):
|
||||
continue
|
||||
start, _, _, characteristics = self.sections[section - 1]
|
||||
if not characteristics & IMAGE_SCN_CNT_CODE:
|
||||
continue
|
||||
if storage == IMAGE_SYM_CLASS_STATIC and (kind >> 4) != 2 and value == 0:
|
||||
continue
|
||||
if name[:4] == b"\0\0\0\0":
|
||||
at = strings + struct.unpack_from("<I", name, 4)[0]
|
||||
end = data.index(b"\0", at)
|
||||
text = data[at:end]
|
||||
else:
|
||||
text = name.rstrip(b"\0")
|
||||
result.append((start + value, text.decode("utf-8", "replace")))
|
||||
return result
|
||||
|
||||
def export_symbols(self, data, rva, size):
|
||||
directory = self.offset(rva)
|
||||
_, names, function_rva, name_rva, ordinal_rva = struct.unpack_from("<IIIII", data, directory + 20)
|
||||
function_table = self.offset(function_rva)
|
||||
result = []
|
||||
if names:
|
||||
name_table = self.offset(name_rva)
|
||||
ordinal_table = self.offset(ordinal_rva)
|
||||
for index in range(names):
|
||||
ordinal = struct.unpack_from("<H", data, ordinal_table + index * 2)[0]
|
||||
address = struct.unpack_from("<I", data, function_table + ordinal * 4)[0]
|
||||
if rva <= address < rva + size:
|
||||
continue
|
||||
at = self.offset(struct.unpack_from("<I", data, name_table + index * 4)[0])
|
||||
result.append((address, data[at:data.index(b"\0", at)].decode("utf-8", "replace")))
|
||||
return result
|
||||
|
||||
def name(self, rva):
|
||||
for start, end, addresses, names in self.code:
|
||||
if start <= rva < end:
|
||||
index = bisect.bisect_right(addresses, rva) - 1
|
||||
return None if index < 0 else ("~" if self.exported else "") + names[index]
|
||||
return None
|
||||
|
||||
|
||||
class Symbolizer:
|
||||
def __init__(self, libs, headers, demangle=False, warn=print, unverified=False):
|
||||
self.files = {}
|
||||
for directory in libs:
|
||||
for path in Path(directory).iterdir():
|
||||
if path.is_file():
|
||||
self.files.setdefault(path.name.lower(), path)
|
||||
self.headers = headers
|
||||
self.modules = {}
|
||||
self.demangle = demangle and shutil.which("c++filt") is not None
|
||||
self.demangled = {}
|
||||
self.warn = warn
|
||||
self.unverified = unverified
|
||||
|
||||
def module(self, name):
|
||||
if name in self.modules:
|
||||
return self.modules[name]
|
||||
module = None
|
||||
path = self.files.get(name.lower())
|
||||
if path is not None:
|
||||
try:
|
||||
module = PeModule(path)
|
||||
except (ValueError, struct.error) as error:
|
||||
self.warn(f"{name}: {error}")
|
||||
if module is not None and name not in self.headers and not self.unverified:
|
||||
self.warn(f"{name}: the samples have no '# module' record for it, so {path} may be another build; its frames stay unsymbolized (--unverified symbolizes them anyway)")
|
||||
module = None
|
||||
if module is not None and name in self.headers and self.headers[name] != (module.size, module.timestamp):
|
||||
size, timestamp = self.headers[name]
|
||||
self.warn(f"{name}: the sampled module had size {size:#x} and timestamp {timestamp:#x}, but {path} has {module.size:#x} and {module.timestamp:#x}; its frames stay unsymbolized")
|
||||
module = None
|
||||
self.modules[name] = module
|
||||
return module
|
||||
|
||||
def frame(self, frame):
|
||||
name, separator, offset = frame.partition("+0x")
|
||||
if not separator:
|
||||
return frame
|
||||
module = self.module(name)
|
||||
if module is None:
|
||||
return frame
|
||||
try:
|
||||
symbol = module.name(int(offset, 16))
|
||||
except ValueError:
|
||||
return frame
|
||||
if symbol is None:
|
||||
return frame
|
||||
return f"{name}!{self.pretty(symbol)}"
|
||||
|
||||
def pretty(self, symbol):
|
||||
if not self.demangle or not symbol.lstrip("~").startswith("_Z"):
|
||||
return symbol
|
||||
if symbol not in self.demangled:
|
||||
plain = symbol.lstrip("~")
|
||||
result = subprocess.run(["c++filt", plain], capture_output=True, text=True)
|
||||
text = result.stdout.strip() if result.returncode == 0 and result.stdout.strip() else plain
|
||||
self.demangled[symbol] = ("~" if symbol.startswith("~") else "") + text
|
||||
return self.demangled[symbol]
|
||||
|
||||
|
||||
def read_headers(lines):
|
||||
headers = {}
|
||||
for line in lines:
|
||||
parts = line.split()
|
||||
if len(parts) == 5 and parts[:2] == ["#", "module"]:
|
||||
headers[parts[2]] = (int(parts[3], 16), int(parts[4], 16))
|
||||
return headers
|
||||
|
||||
|
||||
class Samples:
|
||||
def __init__(self):
|
||||
self.headers = {}
|
||||
self.stacks = collections.Counter()
|
||||
self.self_counts = collections.Counter()
|
||||
self.inclusive = collections.Counter()
|
||||
self.total = collections.Counter()
|
||||
|
||||
|
||||
def load(path, thread=None):
|
||||
lines = Path(path).read_text(encoding="utf-8-sig").splitlines()
|
||||
samples = Samples()
|
||||
samples.headers = read_headers(lines)
|
||||
current = None
|
||||
for number, line in enumerate(lines, 1):
|
||||
parts = line.split()
|
||||
if len(parts) == 3 and parts[0] == "#" and parts[2] == "samples" and parts[1].isdigit():
|
||||
samples.total["samples"] += int(parts[1])
|
||||
continue
|
||||
if line.startswith("#") or not line.strip():
|
||||
continue
|
||||
if len(parts) == 4 and parts[0] == "thread" and parts[2] == "samples" and parts[1].isdigit() and parts[3].isdigit():
|
||||
current = parts[1]
|
||||
if thread is None or current == thread:
|
||||
samples.total["threads"] += int(parts[3])
|
||||
continue
|
||||
if len(parts) == 4 and parts[1].startswith("0x") and parts[2].isdigit() and parts[3].isdigit():
|
||||
if thread is None or current == thread:
|
||||
samples.self_counts[(parts[0], parts[1])] += int(parts[2])
|
||||
samples.inclusive[(parts[0], parts[1])] += int(parts[3])
|
||||
continue
|
||||
frames, _, count = line.rpartition(" ")
|
||||
if not frames or not count.isdigit():
|
||||
raise ValueError(f"{path}:{number}: expected 'frame;...;frame count', 'module offset self inclusive' or 'thread <id> samples <n>'")
|
||||
samples.stacks[frames] += int(count)
|
||||
return samples
|
||||
|
||||
|
||||
def subtract(later, earlier):
|
||||
result = collections.Counter(later)
|
||||
result.subtract(earlier)
|
||||
return collections.Counter({key: count for key, count in result.items() if count > 0})
|
||||
|
||||
|
||||
def window(later, earlier):
|
||||
result = Samples()
|
||||
result.headers = {**earlier.headers, **later.headers}
|
||||
for name in ("stacks", "self_counts", "inclusive", "total"):
|
||||
setattr(result, name, subtract(getattr(later, name), getattr(earlier, name)))
|
||||
return result
|
||||
|
||||
|
||||
def summarize(symbolizer, samples, top):
|
||||
self_counts = collections.Counter()
|
||||
inclusive = collections.Counter()
|
||||
total = 0
|
||||
if samples.stacks:
|
||||
for frames, count in samples.stacks.items():
|
||||
names = [symbolizer.frame(frame) for frame in frames.split(";")]
|
||||
total += count
|
||||
self_counts[names[-1]] += count
|
||||
for name in set(names):
|
||||
inclusive[name] += count
|
||||
else:
|
||||
for (module, offset), count in samples.self_counts.items():
|
||||
self_counts[symbolizer.frame(f"{module}+{offset}")] += count
|
||||
for (module, offset), count in samples.inclusive.items():
|
||||
inclusive[symbolizer.frame(f"{module}+{offset}")] += count
|
||||
total = samples.total["threads"] or samples.total["samples"] or sum(samples.self_counts.values())
|
||||
lines = [f"{total} samples"]
|
||||
for title, counter in (("self", self_counts), ("inclusive", inclusive)):
|
||||
if not counter:
|
||||
continue
|
||||
lines.append(f"hottest by {title}:")
|
||||
for name, count in counter.most_common(top):
|
||||
lines.append(f" {count:8d} {100.0 * count / total if total else 0.0:5.1f}% {name}")
|
||||
return lines
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("samples", help="collapsed-stack file or address file written by a Windows sampler")
|
||||
parser.add_argument("--libs", action="append", required=True, help="directory holding the modules that ran (repeatable)")
|
||||
parser.add_argument("--since", help="an earlier copy of the same file, subtracted to keep the samples taken after it")
|
||||
parser.add_argument("--out", help="write the symbolized collapsed stacks to this file")
|
||||
parser.add_argument("--top", type=int, default=30, help="functions listed per table (default 30)")
|
||||
parser.add_argument("--demangle", action="store_true", help="demangle C++ names with c++filt when it is on PATH")
|
||||
parser.add_argument("--thread", help="keep only this thread of an APS5_SAMPLE_THREADS file")
|
||||
parser.add_argument("--unverified", action="store_true", help="also symbolize modules the samples have no '# module' record for")
|
||||
args = parser.parse_args(argv)
|
||||
samples = load(args.samples, args.thread)
|
||||
if args.since:
|
||||
samples = window(samples, load(args.since, args.thread))
|
||||
symbolizer = Symbolizer(args.libs, samples.headers, args.demangle, warn=lambda text: print("warning: " + text, file=sys.stderr), unverified=args.unverified)
|
||||
for line in summarize(symbolizer, samples, args.top):
|
||||
print(line)
|
||||
if args.out:
|
||||
if not samples.stacks:
|
||||
parser.error("--out needs a collapsed-stack file")
|
||||
with open(args.out, "w", encoding="utf-8") as output:
|
||||
for frames, count in samples.stacks.items():
|
||||
output.write(";".join(symbolizer.frame(frame) for frame in frames.split(";")) + f" {count}\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,202 @@
|
||||
import contextlib
|
||||
import io
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
import symbolize_samples
|
||||
|
||||
TIMESTAMP = 0x6AD92AC4
|
||||
SIZE_OF_IMAGE = 0x3000
|
||||
|
||||
|
||||
def symbol(name, value, section, kind, storage, aux=0):
|
||||
return struct.pack("<8sIhHBB", name, value, section, kind, storage, aux)
|
||||
|
||||
|
||||
def build_pe(path, with_symbols=True, with_exports=False, second_code=False):
|
||||
image = bytearray(0xC00)
|
||||
image[0:2] = b"MZ"
|
||||
struct.pack_into("<I", image, 0x3C, 0x40)
|
||||
image[0x40:0x44] = b"PE\0\0"
|
||||
symbols = b""
|
||||
strings = b""
|
||||
if with_symbols:
|
||||
long_name = b"AgcDriver_long_function_name\0"
|
||||
strings = struct.pack("<I", 4 + len(long_name)) + long_name
|
||||
symbols = b"".join([
|
||||
symbol(b".text", 0, 1, 0, 3, aux=1), b"\0" * 18,
|
||||
symbol(b"short", 0x10, 1, 0x20, 2),
|
||||
symbol(b"\0\0\0\0" + struct.pack("<I", 4), 0x40, 1, 0x20, 2),
|
||||
symbol(b"static_f", 0x80, 1, 0x20, 3),
|
||||
symbol(b"data_obj", 0x08, 2, 0, 2),
|
||||
] + ([symbol(b"late_fn", 0x40, 3, 0x20, 2)] if second_code else []))
|
||||
symbol_count = len(symbols) // 18
|
||||
struct.pack_into("<HHIIIHH", image, 0x44, 0x8664, 3 if second_code else 2, TIMESTAMP, 0x800 if with_symbols else 0, symbol_count, 240, 0x22)
|
||||
optional = 0x58
|
||||
struct.pack_into("<H", image, optional, 0x20B)
|
||||
struct.pack_into("<Q", image, optional + 24, 0x140000000)
|
||||
struct.pack_into("<I", image, optional + 56, SIZE_OF_IMAGE)
|
||||
if with_exports:
|
||||
struct.pack_into("<II", image, optional + 112, 0x2000, 0x100)
|
||||
sections = optional + 240
|
||||
struct.pack_into("<8sIIII", image, sections, b".text", 0x100, 0x1000, 0x100, 0x400)
|
||||
struct.pack_into("<I", image, sections + 36, 0x60000020)
|
||||
struct.pack_into("<8sIIII", image, sections + 40, b".rdata", 0x200, 0x2000, 0x200, 0x600)
|
||||
struct.pack_into("<I", image, sections + 76, 0x40000040)
|
||||
if second_code:
|
||||
struct.pack_into("<8sIIII", image, sections + 80, b".text2", 0x100, 0x2800, 0x100, 0xA00)
|
||||
struct.pack_into("<I", image, sections + 116, 0x60000020)
|
||||
if with_exports:
|
||||
directory = 0x600
|
||||
struct.pack_into("<IIHHIIIIIII", image, directory, 0, 0, 0, 0, 0, 1, 2, 2, 0x2040, 0x2048, 0x2050)
|
||||
struct.pack_into("<II", image, 0x640, 0x1020, 0x1060)
|
||||
struct.pack_into("<II", image, 0x648, 0x2060, 0x2070)
|
||||
struct.pack_into("<HH", image, 0x650, 0, 1)
|
||||
image[0x660:0x660 + 11] = b"first_func\0"
|
||||
image[0x670:0x670 + 12] = b"second_func\0"
|
||||
if with_symbols:
|
||||
image[0x800:0x800 + len(symbols)] = symbols
|
||||
image[0x800 + len(symbols):0x800 + len(symbols) + len(strings)] = strings
|
||||
Path(path).write_bytes(bytes(image))
|
||||
|
||||
|
||||
class SymbolizeSamplesTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.directory.name)
|
||||
self.libs = self.root / "libs"
|
||||
self.libs.mkdir()
|
||||
build_pe(self.libs / "libSceAgcDriver.prx")
|
||||
build_pe(self.libs / "libwinpthread-1.dll", with_symbols=False, with_exports=True)
|
||||
|
||||
def tearDown(self):
|
||||
self.directory.cleanup()
|
||||
|
||||
def test_coff_symbols_name_code_addresses(self):
|
||||
module = symbolize_samples.PeModule(self.libs / "libSceAgcDriver.prx")
|
||||
self.assertEqual((module.size, module.timestamp), (SIZE_OF_IMAGE, TIMESTAMP))
|
||||
self.assertFalse(module.exported)
|
||||
self.assertIsNone(module.name(0x1005))
|
||||
self.assertEqual(module.name(0x1010), "short")
|
||||
self.assertEqual(module.name(0x103F), "short")
|
||||
self.assertEqual(module.name(0x1045), "AgcDriver_long_function_name")
|
||||
self.assertEqual(module.name(0x1085), "static_f")
|
||||
self.assertIsNone(module.name(0x1100))
|
||||
self.assertIsNone(module.name(0x2008))
|
||||
self.assertNotIn("data_obj", module.names)
|
||||
self.assertNotIn(".text", module.names)
|
||||
|
||||
def test_export_table_names_a_module_without_symbols(self):
|
||||
module = symbolize_samples.PeModule(self.libs / "libwinpthread-1.dll")
|
||||
self.assertTrue(module.exported)
|
||||
self.assertEqual(module.name(0x1030), "~first_func")
|
||||
self.assertEqual(module.name(0x1070), "~second_func")
|
||||
|
||||
def test_frames_keep_unknown_modules_and_out_of_range_addresses(self):
|
||||
symbolizer = symbolize_samples.Symbolizer([self.libs], {}, warn=lambda text: None, unverified=True)
|
||||
self.assertEqual(symbolizer.frame("libSceAgcDriver.prx+0x1046"), "libSceAgcDriver.prx!AgcDriver_long_function_name")
|
||||
self.assertEqual(symbolizer.frame("LIBSCEAGCDRIVER.PRX+0x1011"), "LIBSCEAGCDRIVER.PRX!short")
|
||||
self.assertEqual(symbolizer.frame("ntdll.dll+0x10"), "ntdll.dll+0x10")
|
||||
self.assertEqual(symbolizer.frame("libSceAgcDriver.prx+0x10"), "libSceAgcDriver.prx+0x10")
|
||||
self.assertEqual(symbolizer.frame("[truncated]"), "[truncated]")
|
||||
|
||||
def test_a_module_whose_size_or_timestamp_differs_stays_unsymbolized(self):
|
||||
warnings = []
|
||||
headers = {"libSceAgcDriver.prx": (SIZE_OF_IMAGE, TIMESTAMP + 1)}
|
||||
symbolizer = symbolize_samples.Symbolizer([self.libs], headers, warn=warnings.append)
|
||||
self.assertEqual(symbolizer.frame("libSceAgcDriver.prx+0x1011"), "libSceAgcDriver.prx+0x1011")
|
||||
self.assertEqual(len(warnings), 1)
|
||||
self.assertIn("timestamp", warnings[0])
|
||||
matching = symbolize_samples.Symbolizer([self.libs], {"libSceAgcDriver.prx": (SIZE_OF_IMAGE, TIMESTAMP)}, warn=warnings.append)
|
||||
self.assertEqual(matching.frame("libSceAgcDriver.prx+0x1011"), "libSceAgcDriver.prx!short")
|
||||
|
||||
def test_since_subtracts_an_earlier_snapshot_and_out_writes_symbolized_stacks(self):
|
||||
header = f"# samples=9 depth=64 truncated=0\n# module libSceAgcDriver.prx {SIZE_OF_IMAGE:#x} {TIMESTAMP:#x}\n"
|
||||
earlier = self.root / "worker-early.collapsed.txt"
|
||||
later = self.root / "worker-late.collapsed.txt"
|
||||
earlier.write_text(header + "ntdll.dll+0x1;libSceAgcDriver.prx+0x1011 4\nntdll.dll+0x1;libSceAgcDriver.prx+0x1081 2\n", encoding="utf-8")
|
||||
later.write_text(header + "ntdll.dll+0x1;libSceAgcDriver.prx+0x1011 7\nntdll.dll+0x1;libSceAgcDriver.prx+0x1081 2\nntdll.dll+0x1;libSceAgcDriver.prx+0x1041 3\n", encoding="utf-8")
|
||||
out = self.root / "steady.collapsed.txt"
|
||||
printed = io.StringIO()
|
||||
with contextlib.redirect_stdout(printed):
|
||||
self.assertEqual(symbolize_samples.main([str(later), "--since", str(earlier), "--libs", str(self.libs), "--out", str(out), "--top", "5"]), 0)
|
||||
text = printed.getvalue()
|
||||
self.assertIn("6 samples", text)
|
||||
self.assertIn("libSceAgcDriver.prx!short", text)
|
||||
self.assertIn("libSceAgcDriver.prx!AgcDriver_long_function_name", text)
|
||||
self.assertNotIn("static_f", text)
|
||||
written = sorted(out.read_text(encoding="utf-8").splitlines())
|
||||
self.assertEqual(written, ["ntdll.dll+0x1;libSceAgcDriver.prx!AgcDriver_long_function_name 3", "ntdll.dll+0x1;libSceAgcDriver.prx!short 3"])
|
||||
|
||||
def run_main(self, *arguments):
|
||||
printed = io.StringIO()
|
||||
errors = io.StringIO()
|
||||
with contextlib.redirect_stdout(printed), contextlib.redirect_stderr(errors):
|
||||
self.assertEqual(symbolize_samples.main([str(argument) for argument in arguments]), 0)
|
||||
return printed.getvalue().splitlines(), errors.getvalue()
|
||||
|
||||
def test_address_files_are_summarized_by_self_and_inclusive_samples(self):
|
||||
samples = self.root / "worker.txt"
|
||||
samples.write_text(f"# 5 samples\nlibSceAgcDriver.prx 0x1011 3 9\nlibSceAgcDriver.prx 0x1012 1 4\nlibSceAgcDriver.prx 0x1045 0 5\nntdll.dll 0x10 1 1\n# module libSceAgcDriver.prx {SIZE_OF_IMAGE:#x} {TIMESTAMP:#x}\n", encoding="utf-8")
|
||||
lines, _ = self.run_main(samples, "--libs", self.libs)
|
||||
self.assertEqual(lines[0], "5 samples")
|
||||
self.assertIn("4 80.0% libSceAgcDriver.prx!short", lines[2])
|
||||
inclusive = lines[lines.index("hottest by inclusive:") + 1:]
|
||||
self.assertIn("13 260.0% libSceAgcDriver.prx!short", inclusive[0])
|
||||
self.assertIn("5 100.0% libSceAgcDriver.prx!AgcDriver_long_function_name", inclusive[1])
|
||||
|
||||
def test_a_symbol_does_not_reach_into_a_later_code_section(self):
|
||||
build_pe(self.libs / "two.prx", second_code=True)
|
||||
module = symbolize_samples.PeModule(self.libs / "two.prx")
|
||||
self.assertEqual(module.name(0x1085), "static_f")
|
||||
self.assertIsNone(module.name(0x2810))
|
||||
self.assertEqual(module.name(0x2840), "late_fn")
|
||||
self.assertEqual(module.name(0x28F0), "late_fn")
|
||||
|
||||
def test_a_module_without_a_record_stays_unsymbolized_unless_unverified(self):
|
||||
samples = self.root / "worker.txt"
|
||||
samples.write_text("# 4 samples\nlibSceAgcDriver.prx 0x1011 4 4\n", encoding="utf-8")
|
||||
lines, errors = self.run_main(samples, "--libs", self.libs)
|
||||
self.assertIn("libSceAgcDriver.prx+0x1011", lines[2])
|
||||
self.assertIn("no '# module' record", errors)
|
||||
lines, errors = self.run_main(samples, "--libs", self.libs, "--unverified")
|
||||
self.assertIn("libSceAgcDriver.prx!short", lines[2])
|
||||
self.assertEqual(errors, "")
|
||||
|
||||
def test_thread_files_keep_each_heading_and_can_select_one_thread(self):
|
||||
samples = self.root / "threads.txt"
|
||||
record = f"# module libSceAgcDriver.prx {SIZE_OF_IMAGE:#x} {TIMESTAMP:#x}\n"
|
||||
samples.write_text("# 50 rounds\nthread 7 samples 10\n libSceAgcDriver.prx 0x1011 6 8\nthread 9 samples 40\n libSceAgcDriver.prx 0x1045 20 30\n libSceAgcDriver.prx 0x1011 2 12\n" + record, encoding="utf-8")
|
||||
lines, _ = self.run_main(samples, "--libs", self.libs)
|
||||
self.assertEqual(lines[0], "50 samples")
|
||||
self.assertIn("20 40.0% libSceAgcDriver.prx!AgcDriver_long_function_name", lines[2])
|
||||
lines, _ = self.run_main(samples, "--libs", self.libs, "--thread", "7")
|
||||
self.assertEqual(lines[0], "10 samples")
|
||||
self.assertIn("6 60.0% libSceAgcDriver.prx!short", lines[2])
|
||||
self.assertNotIn("AgcDriver_long_function_name", "\n".join(lines))
|
||||
|
||||
def test_since_subtracts_both_counters_of_an_address_file(self):
|
||||
record = f"# module libSceAgcDriver.prx {SIZE_OF_IMAGE:#x} {TIMESTAMP:#x}\n"
|
||||
earlier = self.root / "early.txt"
|
||||
later = self.root / "late.txt"
|
||||
earlier.write_text("# 4 samples\nlibSceAgcDriver.prx 0x1011 4 6\n" + record, encoding="utf-8")
|
||||
later.write_text("# 10 samples\nlibSceAgcDriver.prx 0x1011 7 12\nlibSceAgcDriver.prx 0x1045 3 3\n" + record, encoding="utf-8")
|
||||
lines, _ = self.run_main(later, "--since", earlier, "--libs", self.libs)
|
||||
self.assertEqual(lines[0], "6 samples")
|
||||
self.assertIn("3 50.0% libSceAgcDriver.prx!short", "\n".join(lines[2:4]))
|
||||
inclusive = lines[lines.index("hottest by inclusive:") + 1:]
|
||||
self.assertIn("6 100.0% libSceAgcDriver.prx!short", inclusive[0])
|
||||
|
||||
def test_malformed_lines_are_rejected_with_their_line_number(self):
|
||||
bad = self.root / "bad.collapsed.txt"
|
||||
bad.write_text("# header\nlibSceAgcDriver.prx+0x1011 many\n", encoding="utf-8")
|
||||
with self.assertRaisesRegex(ValueError, r"bad\.collapsed\.txt:2:"):
|
||||
symbolize_samples.load(bad)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user