diff --git a/CtfPlayback/EventStreams/CtfEvent.cs b/CtfPlayback/EventStreams/CtfEvent.cs index 73aa195..0be1fbb 100644 --- a/CtfPlayback/EventStreams/CtfEvent.cs +++ b/CtfPlayback/EventStreams/CtfEvent.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using CtfPlayback.EventStreams.Interfaces; using CtfPlayback.FieldValues; +using CtfPlayback.Metadata; using CtfPlayback.Metadata.Interfaces; namespace CtfPlayback.EventStreams @@ -25,6 +26,8 @@ public CtfEvent(PacketReader packetReader, ICtfMetadata metadata, ICtfPacket own public ulong ByteOffsetWithinPacket { get; private set; } + public uint StreamId => this.owningPacket.StreamId; + public CtfTimestamp Timestamp { get; private set; } public CtfFieldValue StreamDefinedEventHeader { get; private set; } @@ -102,8 +105,7 @@ public void Read() private void ReadStreamEventHeader() { - uint streamIndex = this.owningPacket.StreamId; - var eventHeaderDescriptor = this.packetReader.Metadata.Streams[(int)streamIndex].EventHeader; + var eventHeaderDescriptor = this.packetReader.Metadata.GetStream(this.owningPacket.StreamId).EventHeader; Debug.Assert(eventHeaderDescriptor != null); StreamDefinedEventHeader = eventHeaderDescriptor.Read(this.packetReader); @@ -111,8 +113,7 @@ private void ReadStreamEventHeader() private void ReadStreamEventContext() { - uint streamIndex = this.owningPacket.StreamId; - var eventContextDescriptor = this.packetReader.Metadata.Streams[(int)streamIndex].EventContext; + var eventContextDescriptor = this.packetReader.Metadata.GetStream(this.owningPacket.StreamId).EventContext; if (eventContextDescriptor == null) { return; diff --git a/CtfPlayback/EventStreams/CtfPacket.cs b/CtfPlayback/EventStreams/CtfPacket.cs index a7c9d00..39a59f5 100644 --- a/CtfPlayback/EventStreams/CtfPacket.cs +++ b/CtfPlayback/EventStreams/CtfPacket.cs @@ -6,6 +6,7 @@ using System.IO; using CtfPlayback.EventStreams.Interfaces; using CtfPlayback.FieldValues; +using CtfPlayback.Metadata; using CtfPlayback.Metadata.Interfaces; namespace CtfPlayback.EventStreams @@ -61,10 +62,12 @@ public bool ReadPacketMetadata() this.PacketByteOffset = this.packetReader.CountOfBytesProcessed; ReadPacketHeader(); - ReadPacketContext(); + // The packet context layout depends on the stream, which is identified in the packet header. this.DetermineStreamIndex(); + ReadPacketContext(); + this.packetReader.SetPacketSize(this.packetReader.PlaybackCustomization.GetBitsInPacket(this)); this.BitsInPacketContent = @@ -116,7 +119,7 @@ private void ReadPacketHeader() private void ReadPacketContext() { - this.StreamPacketContext = this.Metadata.Streams[(int)this.StreamId].PacketContext.Read(this.packetReader) as CtfStructValue; + this.StreamPacketContext = this.Metadata.GetStream(this.StreamId).PacketContext.Read(this.packetReader) as CtfStructValue; } private ulong GetRemainingBits() @@ -135,6 +138,7 @@ private void DetermineStreamIndex() if (streamIdDescriptor == null) { this.StreamId = 0; + return; } bool streamIdFound = this.TracePacketHeader.FieldsByName.TryGetValue("stream_id", out var streamIdFieldValue); diff --git a/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs b/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs index c80a429..3775772 100644 --- a/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs +++ b/CtfPlayback/EventStreams/Interfaces/ICtfEvent.cs @@ -16,6 +16,11 @@ public interface ICtfEvent /// ulong ByteOffsetWithinPacket { get; } + /// + /// Id of the stream (stream class) that contains this event. + /// + uint StreamId { get; } + /// /// Timestamp associated with the event /// diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2Json.cs b/CtfPlayback/Metadata/Ctf2/Ctf2Json.cs new file mode 100644 index 0000000..9e0cf89 --- /dev/null +++ b/CtfPlayback/Metadata/Ctf2/Ctf2Json.cs @@ -0,0 +1,273 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +namespace CtfPlayback.Metadata.Ctf2 +{ + /// + /// A JSON number, kept in its textual form so that 64-bit unsigned values are not truncated. + /// + internal sealed class Ctf2JsonNumber + { + internal Ctf2JsonNumber(string text) + { + this.Text = text; + } + + internal string Text { get; } + + internal bool IsNegative => this.Text.StartsWith("-", StringComparison.Ordinal); + + internal long AsLong() => long.Parse(this.Text, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture); + + internal ulong AsULong() => ulong.Parse(this.Text, NumberStyles.None, CultureInfo.InvariantCulture); + + public override string ToString() => this.Text; + } + + /// + /// Minimal JSON reader used for CTF 2 metadata. Objects are returned as + /// of string to object, arrays as of object, + /// numbers as , and strings, booleans and null as their .NET equivalents. + /// + internal sealed class Ctf2Json + { + private readonly string text; + private int position; + + private Ctf2Json(string text) + { + this.text = text; + } + + internal static object Parse(string text) + { + var reader = new Ctf2Json(text); + reader.SkipWhitespace(); + var value = reader.ReadValue(); + reader.SkipWhitespace(); + if (reader.position != text.Length) + { + throw reader.Error("Unexpected trailing characters"); + } + + return value; + } + + private object ReadValue() + { + if (this.position >= this.text.Length) + { + throw this.Error("Unexpected end of JSON"); + } + + char c = this.text[this.position]; + switch (c) + { + case '{': + return this.ReadObject(); + case '[': + return this.ReadArray(); + case '"': + return this.ReadString(); + case 't': + this.Expect("true"); + return true; + case 'f': + this.Expect("false"); + return false; + case 'n': + this.Expect("null"); + return null; + default: + if (c == '-' || (c >= '0' && c <= '9')) + { + return this.ReadNumber(); + } + + throw this.Error($"Unexpected character '{c}'"); + } + } + + private Dictionary ReadObject() + { + var result = new Dictionary(StringComparer.Ordinal); + this.position++; + this.SkipWhitespace(); + if (this.TryConsume('}')) + { + return result; + } + + while (true) + { + this.SkipWhitespace(); + if (this.position >= this.text.Length || this.text[this.position] != '"') + { + throw this.Error("Expected property name"); + } + + string name = this.ReadString(); + this.SkipWhitespace(); + if (!this.TryConsume(':')) + { + throw this.Error("Expected ':'"); + } + + this.SkipWhitespace(); + result[name] = this.ReadValue(); + this.SkipWhitespace(); + + if (this.TryConsume(',')) + { + continue; + } + + if (this.TryConsume('}')) + { + return result; + } + + throw this.Error("Expected ',' or '}'"); + } + } + + private List ReadArray() + { + var result = new List(); + this.position++; + this.SkipWhitespace(); + if (this.TryConsume(']')) + { + return result; + } + + while (true) + { + this.SkipWhitespace(); + result.Add(this.ReadValue()); + this.SkipWhitespace(); + + if (this.TryConsume(',')) + { + continue; + } + + if (this.TryConsume(']')) + { + return result; + } + + throw this.Error("Expected ',' or ']'"); + } + } + + private string ReadString() + { + this.position++; + var sb = new StringBuilder(); + while (this.position < this.text.Length) + { + char c = this.text[this.position++]; + if (c == '"') + { + return sb.ToString(); + } + + if (c != '\\') + { + sb.Append(c); + continue; + } + + if (this.position >= this.text.Length) + { + break; + } + + char escaped = this.text[this.position++]; + switch (escaped) + { + case '"': sb.Append('"'); break; + case '\\': sb.Append('\\'); break; + case '/': sb.Append('/'); break; + case 'b': sb.Append('\b'); break; + case 'f': sb.Append('\f'); break; + case 'n': sb.Append('\n'); break; + case 'r': sb.Append('\r'); break; + case 't': sb.Append('\t'); break; + case 'u': + if (this.position + 4 > this.text.Length || + !int.TryParse(this.text.Substring(this.position, 4), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out int codeUnit)) + { + throw this.Error("Invalid unicode escape"); + } + + sb.Append((char)codeUnit); + this.position += 4; + break; + default: + throw this.Error($"Invalid escape character '{escaped}'"); + } + } + + throw this.Error("Unterminated string"); + } + + private Ctf2JsonNumber ReadNumber() + { + int start = this.position; + while (this.position < this.text.Length) + { + char c = this.text[this.position]; + if ((c >= '0' && c <= '9') || c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E') + { + this.position++; + continue; + } + + break; + } + + return new Ctf2JsonNumber(this.text.Substring(start, this.position - start)); + } + + private void Expect(string literal) + { + if (string.CompareOrdinal(this.text, this.position, literal, 0, literal.Length) != 0) + { + throw this.Error($"Expected '{literal}'"); + } + + this.position += literal.Length; + } + + private bool TryConsume(char c) + { + if (this.position < this.text.Length && this.text[this.position] == c) + { + this.position++; + return true; + } + + return false; + } + + private void SkipWhitespace() + { + while (this.position < this.text.Length && char.IsWhiteSpace(this.text[this.position])) + { + this.position++; + } + } + + private CtfMetadataException Error(string message) + { + return new CtfMetadataException($"Invalid CTF 2 metadata JSON: {message} at offset {this.position}."); + } + } +} diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs b/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs new file mode 100644 index 0000000..306606e --- /dev/null +++ b/CtfPlayback/Metadata/Ctf2/Ctf2MetadataParser.cs @@ -0,0 +1,792 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using CtfPlayback.Metadata.Helpers; +using CtfPlayback.Metadata.Interfaces; +using CtfPlayback.Metadata.InternalHelpers; +using CtfPlayback.Metadata.NamedScopes; +using CtfPlayback.Metadata.TypeInterfaces; +using CtfPlayback.Metadata.Types; + +namespace CtfPlayback.Metadata.Ctf2 +{ + /// + /// Parses CTF 2 metadata (a JSON text sequence of fragments, see https://diamon.org/ctf/) into the same + /// descriptor model used for CTF 1.8, so that the rest of the playback pipeline is format-agnostic. + /// + public sealed class Ctf2MetadataParser + : ICtfMetadataParser + { + /// + /// Each CTF 2 metadata fragment is preceded by this record separator (RFC 7464). + /// + public const char RecordSeparator = '\u001e'; + + private const string DefaultClockTimestampRole = "default-clock-timestamp"; + private const string PacketEndDefaultClockTimestampRole = "packet-end-default-clock-timestamp"; + + private readonly ICtfMetadataBuilder metadataBuilder; + private readonly bool prefixEventFieldNamesWithUnderscore; + + private readonly Dictionary aliases = new Dictionary(StringComparer.Ordinal); + + /// + /// Constructor. + /// + /// The object used to build metadata + /// + /// CTF 1.8 LTTng traces prefix event context and payload field names with '_'. Set this to produce + /// the same field names from CTF 2 metadata. + /// + public Ctf2MetadataParser(ICtfMetadataBuilder metadataBuilder, bool prefixEventFieldNamesWithUnderscore) + { + this.metadataBuilder = metadataBuilder ?? throw new ArgumentNullException(nameof(metadataBuilder)); + this.prefixEventFieldNamesWithUnderscore = prefixEventFieldNamesWithUnderscore; + } + + /// + /// Determines whether the metadata text is CTF 2 metadata. + /// + /// Metadata text, with any packetization removed + /// true if the text is a CTF 2 JSON text sequence + public static bool IsCtf2Metadata(string metadataText) + { + if (metadataText == null) + { + return false; + } + + foreach (char c in metadataText) + { + if (c == RecordSeparator) + { + return true; + } + + if (!char.IsWhiteSpace(c)) + { + return false; + } + } + + return false; + } + + /// + public ICtfMetadata Parse(Stream metadataStream) + { + if (metadataStream == null) + { + throw new ArgumentNullException(nameof(metadataStream)); + } + + string text = CtfMetadataText.Read(CtfMetadataText.ReadAllBytes(metadataStream)); + return this.Parse(text); + } + + /// + /// Parses CTF 2 metadata text. + /// + /// Metadata text, with any packetization removed + /// The parsed metadata + public ICtfMetadata Parse(string metadataText) + { + var fragments = metadataText + .Split(RecordSeparator) + .Where(fragment => !string.IsNullOrWhiteSpace(fragment)) + .Select(fragment => Ctf2Json.Parse(fragment) as Dictionary + ?? throw new CtfMetadataException("CTF 2 metadata fragment is not a JSON object.")) + .ToList(); + + if (fragments.Count == 0 || GetString(fragments[0], "type") != "preamble") + { + throw new CtfMetadataException("CTF 2 metadata does not start with a preamble fragment."); + } + + var preamble = fragments[0]; + if (!TryGetNumber(preamble, "version", out var version) || version.AsLong() != 2) + { + throw new CtfMetadataException($"Unsupported CTF metadata version: {GetValue(preamble, "version")}."); + } + + Dictionary traceClass = null; + var clockClasses = new List>(); + var dataStreamClasses = new List>(); + var eventRecordClasses = new List>(); + + foreach (var fragment in fragments.Skip(1)) + { + string type = GetString(fragment, "type"); + switch (type) + { + case "field-class-alias": + this.aliases[GetRequiredString(fragment, "name")] = GetRequiredValue(fragment, "field-class"); + break; + case "trace-class": + traceClass = fragment; + break; + case "clock-class": + clockClasses.Add(fragment); + break; + case "data-stream-class": + dataStreamClasses.Add(fragment); + break; + case "event-record-class": + eventRecordClasses.Add(fragment); + break; + case "preamble": + throw new CtfMetadataException("CTF 2 metadata contains more than one preamble fragment."); + default: + // Unknown fragment types are ignored so newer metadata can still be read. + break; + } + } + + if (traceClass == null) + { + throw new CtfMetadataException("CTF 2 metadata does not contain a trace-class fragment."); + } + + this.AddTraceClass(traceClass, preamble); + + foreach (var clockClass in clockClasses) + { + this.metadataBuilder.AddClock(CreateClock(clockClass)); + } + + // Streams are looked up by id, and CTF 1.8 streams are declared in id order. Keep that ordering. + foreach (var dataStreamClass in dataStreamClasses.OrderBy(GetDataStreamClassId)) + { + this.AddDataStreamClass(dataStreamClass); + } + + foreach (var eventRecordClass in eventRecordClasses) + { + this.AddEventRecordClass(eventRecordClass); + } + + return this.metadataBuilder; + } + + private void AddTraceClass(Dictionary traceClass, Dictionary preamble) + { + var packetHeaderClass = GetValue(traceClass, "packet-header-field-class"); + if (packetHeaderClass == null) + { + throw new CtfMetadataException("CTF 2 trace-class does not define a packet header field class."); + } + + if (!(this.BuildFieldClass(packetHeaderClass, BuildContext.Header(null)) is CtfStructDescriptor packetHeader)) + { + throw new CtfMetadataException("CTF 2 trace-class packet header field class is not a structure."); + } + + var traceBag = new CtfPropertyBag(); + traceBag.AddValue("major", "2"); + traceBag.AddValue("minor", "0"); + traceBag.AddValue("byte_order", "le"); + string uid = GetString(traceClass, "uid") ?? FormatUuid(GetValue(preamble, "uuid")); + if (uid != null && Guid.TryParse(uid, out _)) + { + traceBag.AddValue("uuid", uid); + } + + this.metadataBuilder.SetTraceDescriptor(new CtfTraceDescriptor(traceBag, packetHeader)); + + var environmentBag = new CtfPropertyBag(); + if (GetValue(traceClass, "environment") is Dictionary environment) + { + foreach (var entry in environment) + { + environmentBag.AddValue(entry.Key, Convert.ToString(entry.Value, CultureInfo.InvariantCulture)); + } + } + + this.metadataBuilder.SetEnvironmentDescriptor(new CtfEnvironmentDescriptor(environmentBag)); + } + + private static CtfClockDescriptor CreateClock(Dictionary clockClass) + { + // CTF 2 data stream classes refer to clock classes by id; older drafts only had a name. + string clockName = GetString(clockClass, "id") ?? GetRequiredString(clockClass, "name"); + + ulong frequency = TryGetNumber(clockClass, "frequency", out var frequencyNumber) ? frequencyNumber.AsULong() : 1000000000ul; + + var bag = new CtfPropertyBag(); + bag.AddValue("name", clockName); + bag.AddValue("freq", frequency.ToString(CultureInfo.InvariantCulture)); + + string description = GetString(clockClass, "description"); + if (description != null) + { + bag.AddValue("description", description); + } + + string uid = GetString(clockClass, "uid") ?? FormatUuid(GetValue(clockClass, "uuid")); + if (uid != null && Guid.TryParse(uid, out _)) + { + bag.AddValue("uuid", uid); + } + + if (TryGetNumber(clockClass, "precision", out var precision) && !precision.IsNegative) + { + bag.AddValue("precision", precision.Text); + } + + if (GetValue(clockClass, "offset-from-origin") is Dictionary offset) + { + // The descriptor expects the full offset in clock cycles. + decimal cycles = 0; + if (TryGetNumber(offset, "seconds", out var seconds)) + { + cycles += decimal.Parse(seconds.Text, CultureInfo.InvariantCulture) * frequency; + } + + if (TryGetNumber(offset, "cycles", out var offsetCycles)) + { + cycles += decimal.Parse(offsetCycles.Text, CultureInfo.InvariantCulture); + } + + if (cycles > 0) + { + bag.AddValue("offset", decimal.ToUInt64(cycles).ToString(CultureInfo.InvariantCulture)); + } + } + + bag.AddValue("absolute", GetValue(clockClass, "origin") != null ? "true" : "false"); + + return new CtfClockDescriptor(bag); + } + + private void AddDataStreamClass(Dictionary dataStreamClass) + { + uint id = GetDataStreamClassId(dataStreamClass); + string clockName = GetString(dataStreamClass, "default-clock-class-id"); + + var packetContextClass = GetValue(dataStreamClass, "packet-context-field-class") + ?? throw new CtfMetadataException($"CTF 2 data stream class {id} does not define a packet context."); + var eventHeaderClass = GetValue(dataStreamClass, "event-record-header-field-class") + ?? throw new CtfMetadataException($"CTF 2 data stream class {id} does not define an event record header."); + + var headerContext = BuildContext.Header(clockName); + + if (!(this.BuildFieldClass(packetContextClass, headerContext) is CtfStructDescriptor packetContext)) + { + throw new CtfMetadataException($"CTF 2 data stream class {id} packet context is not a structure."); + } + + if (!(this.BuildFieldClass(eventHeaderClass, headerContext) is CtfStructDescriptor eventHeader)) + { + throw new CtfMetadataException($"CTF 2 data stream class {id} event record header is not a structure."); + } + + CtfStructDescriptor eventContext = null; + var eventContextClass = GetValue(dataStreamClass, "event-record-common-context-field-class"); + if (eventContextClass != null) + { + eventContext = this.BuildFieldClass(eventContextClass, this.EventContext(clockName)) as CtfStructDescriptor + ?? throw new CtfMetadataException($"CTF 2 data stream class {id} event record common context is not a structure."); + } + + var bag = new CtfPropertyBag(); + bag.AddValue("id", id.ToString(CultureInfo.InvariantCulture)); + + this.metadataBuilder.AddStream(new CtfStreamDescriptor(bag, eventHeader, eventContext, packetContext)); + } + + private void AddEventRecordClass(Dictionary eventRecordClass) + { + string name = GetString(eventRecordClass, "name") ?? string.Empty; + try + { + uint streamId = TryGetNumber(eventRecordClass, "data-stream-class-id", out var streamIdNumber) ? (uint)streamIdNumber.AsULong() : 0; + ulong id = TryGetNumber(eventRecordClass, "id", out var idNumber) ? idNumber.AsULong() : 0; + + var assignments = new Dictionary(StringComparer.Ordinal) + { + ["name"] = name, + ["id"] = id.ToString(CultureInfo.InvariantCulture), + ["stream_id"] = streamId.ToString(CultureInfo.InvariantCulture), + }; + + if (TryGetLogLevel(eventRecordClass, out string logLevel)) + { + assignments["loglevel"] = logLevel; + } + + var context = this.EventContext(null); + var typeDeclarations = new Dictionary(StringComparer.Ordinal); + + var payloadClass = GetValue(eventRecordClass, "payload-field-class"); + typeDeclarations["fields"] = payloadClass != null + ? this.BuildFieldClass(payloadClass, context) + : new CtfStructDescriptor(new CtfPropertyBag(), Array.Empty()); + + var specificContextClass = GetValue(eventRecordClass, "specific-context-field-class"); + if (specificContextClass != null) + { + typeDeclarations["context"] = this.BuildFieldClass(specificContextClass, context); + } + + this.metadataBuilder.AddEvent(assignments, typeDeclarations); + } + catch (Exception e) when (!(e is CtfMetadataException)) + { + throw new CtfMetadataException($"Unable to process CTF 2 event record class '{name}': {e.Message}", e); + } + } + + private BuildContext EventContext(string clockName) + { + return new BuildContext(clockName, this.prefixEventFieldNamesWithUnderscore); + } + + private CtfMetadataTypeDescriptor BuildFieldClass(object fieldClass, BuildContext context, IReadOnlyList selectorOptions = null) + { + var fieldClassObject = this.ResolveFieldClass(fieldClass); + string type = GetRequiredString(fieldClassObject, "type"); + + switch (type) + { + case "fixed-length-unsigned-integer": + case "fixed-length-bit-array": + case "fixed-length-bit-map": + case "fixed-length-boolean": + return BuildFixedLengthInteger(fieldClassObject, false, context, selectorOptions); + + case "fixed-length-signed-integer": + return BuildFixedLengthInteger(fieldClassObject, true, context, selectorOptions); + + case "variable-length-unsigned-integer": + case "variable-length-signed-integer": + if (selectorOptions != null) + { + throw new CtfMetadataException("Variable-length integer variant selectors are not supported."); + } + + return new Ctf2VariableLengthIntegerDescriptor( + type == "variable-length-signed-integer", + GetDisplayBase(fieldClassObject), + GetClockMap(fieldClassObject, context)); + + case "fixed-length-floating-point-number": + case "fixed-length-floating-point": + return BuildFloatingPoint(fieldClassObject); + + case "null-terminated-string": + EnsureUtf8(fieldClassObject); + var stringBag = new CtfPropertyBag(); + stringBag.AddValue("encoding", "UTF8"); + return new CtfStringDescriptor(stringBag); + + case "static-length-string": + EnsureUtf8(fieldClassObject); + return new CtfArrayDescriptor(CreateByte("UTF8", 10), GetRequiredNumber(fieldClassObject, "length").AsULong().ToString(CultureInfo.InvariantCulture)); + + case "dynamic-length-string": + EnsureUtf8(fieldClassObject); + return new CtfArrayDescriptor(CreateByte("UTF8", 10), this.GetFieldLocation(fieldClassObject, "length-field-location", context)); + + case "static-length-blob": + return new CtfArrayDescriptor(CreateByte(null, 16), GetRequiredNumber(fieldClassObject, "length").AsULong().ToString(CultureInfo.InvariantCulture)); + + case "dynamic-length-blob": + return new CtfArrayDescriptor(CreateByte(null, 16), this.GetFieldLocation(fieldClassObject, "length-field-location", context)); + + case "static-length-array": + return new CtfArrayDescriptor( + this.BuildArrayElement(fieldClassObject, context), + GetRequiredNumber(fieldClassObject, "length").AsULong().ToString(CultureInfo.InvariantCulture)); + + case "dynamic-length-array": + return new CtfArrayDescriptor( + this.BuildArrayElement(fieldClassObject, context), + this.GetFieldLocation(fieldClassObject, "length-field-location", context)); + + case "structure": + return this.BuildStructure(fieldClassObject, context); + + case "variant": + return this.BuildVariant(fieldClassObject, context); + + default: + throw new CtfMetadataException($"CTF 2 field class type '{type}' is not supported."); + } + } + + private Dictionary ResolveFieldClass(object fieldClass) + { + var seen = new HashSet(StringComparer.Ordinal); + while (fieldClass is string aliasName) + { + if (!seen.Add(aliasName) || !this.aliases.TryGetValue(aliasName, out fieldClass)) + { + throw new CtfMetadataException($"Unknown CTF 2 field class alias '{aliasName}'."); + } + } + + return fieldClass as Dictionary + ?? throw new CtfMetadataException("CTF 2 field class is not a JSON object."); + } + + private static CtfMetadataTypeDescriptor BuildFixedLengthInteger( + Dictionary fieldClass, + bool signed, + BuildContext context, + IReadOnlyList selectorOptions) + { + // Like the CTF 1.8 path, only whole-byte big-endian integers are byte swapped when read; sub-byte + // big-endian bit fields (e.g. IPv4 header fields in lttng-modules net_* events) are still readable. + bool bigEndian = GetString(fieldClass, "byte-order") == "big-endian"; + var length = GetRequiredNumber(fieldClass, "length"); + + var bag = new CtfPropertyBag(); + bag.AddValue("size", length.Text); + bag.AddValue("align", GetAlignment(fieldClass, "alignment").ToString(CultureInfo.InvariantCulture)); + bag.AddValue("signed", signed ? "true" : "false"); + bag.AddValue("byte_order", bigEndian ? "be" : "le"); + bag.AddValue("base", GetDisplayBase(fieldClass).ToString(CultureInfo.InvariantCulture)); + + string map = GetClockMap(fieldClass, context); + if (map != null) + { + bag.AddValue("map", map); + } + + var integer = new CtfIntegerDescriptor(bag); + if (selectorOptions == null) + { + return integer; + } + + // Variant selection in this library is done through an enumeration whose labels are the option names. + var enumeration = new CtfEnumDescriptor(integer); + foreach (var option in selectorOptions) + { + foreach (var range in option.Ranges) + { + var begin = CreateIntegerLiteral(range.Item1, signed); + var end = CreateIntegerLiteral(range.Item2, signed); + if (!enumeration.AddRange(option.Name, new CtfIntegerRange(integer, begin, end))) + { + throw new CtfMetadataException($"Unable to add variant selector range for option '{option.Name}'."); + } + } + } + + return enumeration; + } + + private static CtfMetadataTypeDescriptor BuildFloatingPoint(Dictionary fieldClass) + { + var bag = new CtfPropertyBag(); + switch (GetRequiredNumber(fieldClass, "length").AsLong()) + { + case 32: + bag.AddValue("exp_dig", "8"); + bag.AddValue("mant_dig", "24"); + break; + case 64: + bag.AddValue("exp_dig", "11"); + bag.AddValue("mant_dig", "53"); + break; + default: + throw new CtfMetadataException("Only 32-bit and 64-bit CTF 2 floating point numbers are supported."); + } + + bag.AddValue("align", GetAlignment(fieldClass, "alignment").ToString(CultureInfo.InvariantCulture)); + bag.AddValue("byte_order", GetString(fieldClass, "byte-order") == "big-endian" ? "be" : "le"); + return new CtfFloatingPointDescriptor(bag); + } + + private CtfMetadataTypeDescriptor BuildArrayElement(Dictionary fieldClass, BuildContext context) + { + var element = this.BuildFieldClass(GetRequiredValue(fieldClass, "element-field-class"), context); + if (TryGetNumber(fieldClass, "minimum-alignment", out var minimumAlignment) && minimumAlignment.AsLong() > element.Align) + { + throw new CtfMetadataException("CTF 2 arrays with a minimum alignment greater than their element alignment are not supported."); + } + + return element; + } + + private CtfStructDescriptor BuildStructure(Dictionary fieldClass, BuildContext context) + { + var members = GetValue(fieldClass, "member-classes") as List ?? new List(); + var memberObjects = members.Select(member => member as Dictionary + ?? throw new CtfMetadataException("CTF 2 structure member class is not a JSON object.")).ToList(); + + // A variant selects its option through a sibling integer field; collect the option ranges so that + // the selector can be built as an enumeration. + var selectorOptionsByMember = new Dictionary>(StringComparer.Ordinal); + foreach (var member in memberObjects) + { + var memberClass = this.ResolveFieldClass(GetRequiredValue(member, "field-class")); + if (GetString(memberClass, "type") != "variant") + { + continue; + } + + string selectorName = GetSiblingSelectorName(memberClass); + if (!selectorOptionsByMember.TryGetValue(selectorName, out var selectorOptions)) + { + selectorOptions = new List(); + selectorOptionsByMember.Add(selectorName, selectorOptions); + } + + selectorOptions.AddRange(GetVariantOptions(memberClass, context)); + } + + var fields = new List(memberObjects.Count); + foreach (var member in memberObjects) + { + string memberName = GetRequiredString(member, "name"); + selectorOptionsByMember.TryGetValue(memberName, out var selectorOptions); + var memberType = this.BuildFieldClass(GetRequiredValue(member, "field-class"), context, selectorOptions); + fields.Add(new CtfFieldDescriptor(memberType, context.MapName(memberName))); + } + + var bag = new CtfPropertyBag(); + bag.AddValue("align", GetAlignment(fieldClass, "minimum-alignment").ToString(CultureInfo.InvariantCulture)); + return new CtfStructDescriptor(bag, fields.ToArray()); + } + + private CtfVariantDescriptor BuildVariant(Dictionary fieldClass, BuildContext context) + { + string selectorName = context.MapName(GetSiblingSelectorName(fieldClass)); + + var options = GetValue(fieldClass, "options") as List + ?? throw new CtfMetadataException("CTF 2 variant does not define any options."); + + var fields = new List(options.Count); + for (int index = 0; index < options.Count; index++) + { + var option = options[index] as Dictionary + ?? throw new CtfMetadataException("CTF 2 variant option is not a JSON object."); + var optionType = this.BuildFieldClass(GetRequiredValue(option, "field-class"), context); + fields.Add(new CtfFieldDescriptor(optionType, GetOptionName(option, index, context))); + } + + return new CtfVariantDescriptor(selectorName, fields); + } + + private static IEnumerable GetVariantOptions(Dictionary variantClass, BuildContext context) + { + var options = GetValue(variantClass, "options") as List + ?? throw new CtfMetadataException("CTF 2 variant does not define any options."); + + for (int index = 0; index < options.Count; index++) + { + var option = (Dictionary)options[index]; + var ranges = GetValue(option, "selector-field-ranges") as List + ?? throw new CtfMetadataException("CTF 2 variant option does not define selector field ranges."); + + var optionRanges = new List>(); + foreach (var range in ranges) + { + if (!(range is List bounds) || bounds.Count != 2 || + !(bounds[0] is Ctf2JsonNumber lower) || !(bounds[1] is Ctf2JsonNumber upper)) + { + throw new CtfMetadataException("CTF 2 variant selector field range is invalid."); + } + + optionRanges.Add(Tuple.Create(lower, upper)); + } + + yield return new SelectorOption(GetOptionName(option, index, context), optionRanges); + } + } + + private static string GetOptionName(Dictionary option, int index, BuildContext context) + { + string name = GetString(option, "name"); + return name == null ? $"option{index}" : context.MapName(name); + } + + private static string GetSiblingSelectorName(Dictionary variantClass) + { + var path = GetFieldLocationPath(variantClass, "selector-field-location"); + if (path.Count != 1) + { + throw new CtfMetadataException("Only CTF 2 variants whose selector is a sibling field are supported."); + } + + return path[0]; + } + + private string GetFieldLocation(Dictionary fieldClass, string property, BuildContext context) + { + return string.Join(".", GetFieldLocationPath(fieldClass, property).Select(context.MapName)); + } + + private static List GetFieldLocationPath(Dictionary fieldClass, string property) + { + if (!(GetValue(fieldClass, property) is Dictionary location) || + !(GetValue(location, "path") is List path) || + path.Count == 0) + { + throw new CtfMetadataException($"CTF 2 field class has an invalid '{property}'."); + } + + return path.Select(element => element as string + ?? throw new CtfMetadataException($"CTF 2 '{property}' path elements must be field names.")).ToList(); + } + + private static CtfIntegerDescriptor CreateByte(string encoding, int displayBase) + { + var bag = new CtfPropertyBag(); + bag.AddValue("size", "8"); + bag.AddValue("align", "8"); + bag.AddValue("signed", "false"); + bag.AddValue("byte_order", "le"); + bag.AddValue("base", displayBase.ToString(CultureInfo.InvariantCulture)); + if (encoding != null) + { + bag.AddValue("encoding", encoding); + } + + return new CtfIntegerDescriptor(bag); + } + + private static string GetClockMap(Dictionary fieldClass, BuildContext context) + { + if (context.ClockName == null || !(GetValue(fieldClass, "roles") is List roles)) + { + return null; + } + + bool isTimestamp = roles.OfType().Any(role => + role == DefaultClockTimestampRole || role == PacketEndDefaultClockTimestampRole); + + return isTimestamp ? $"clock.{context.ClockName}.value" : null; + } + + private static int GetDisplayBase(Dictionary fieldClass) + { + return TryGetNumber(fieldClass, "preferred-display-base", out var displayBase) ? (int)displayBase.AsLong() : 10; + } + + private static long GetAlignment(Dictionary fieldClass, string property) + { + return TryGetNumber(fieldClass, property, out var alignment) ? alignment.AsLong() : 1; + } + + private static void EnsureUtf8(Dictionary fieldClass) + { + string encoding = GetString(fieldClass, "encoding"); + if (encoding != null && encoding != "utf-8") + { + throw new CtfMetadataException($"CTF 2 string encoding '{encoding}' is not supported."); + } + } + + private static IntegerLiteral CreateIntegerLiteral(Ctf2JsonNumber number, bool signed) + { + return signed ? new IntegerLiteral(number.AsLong()) : new IntegerLiteral(number.AsULong()); + } + + private static bool TryGetLogLevel(Dictionary eventRecordClass, out string logLevel) + { + logLevel = null; + if (!(GetValue(eventRecordClass, "attributes") is Dictionary attributes)) + { + return false; + } + + foreach (var namespacedAttributes in attributes.Values.OfType>()) + { + if (TryGetNumber(namespacedAttributes, "log-level", out var level) && !level.IsNegative) + { + logLevel = level.Text; + return true; + } + } + + return false; + } + + private static uint GetDataStreamClassId(Dictionary dataStreamClass) + { + return TryGetNumber(dataStreamClass, "id", out var id) ? (uint)id.AsULong() : 0; + } + + private static string FormatUuid(object uuid) + { + if (!(uuid is List bytes) || bytes.Count != 16 || !bytes.All(b => b is Ctf2JsonNumber)) + { + return null; + } + + var hex = string.Concat(bytes.Cast().Select(b => b.AsULong().ToString("x2", CultureInfo.InvariantCulture))); + return $"{hex.Substring(0, 8)}-{hex.Substring(8, 4)}-{hex.Substring(12, 4)}-{hex.Substring(16, 4)}-{hex.Substring(20)}"; + } + + private static object GetValue(Dictionary json, string property) + { + return json.TryGetValue(property, out var value) ? value : null; + } + + private static object GetRequiredValue(Dictionary json, string property) + { + return GetValue(json, property) ?? throw new CtfMetadataException($"CTF 2 metadata is missing the required '{property}' property."); + } + + private static string GetString(Dictionary json, string property) + { + return GetValue(json, property) as string; + } + + private static string GetRequiredString(Dictionary json, string property) + { + return GetString(json, property) ?? throw new CtfMetadataException($"CTF 2 metadata is missing the required '{property}' string property."); + } + + private static bool TryGetNumber(Dictionary json, string property, out Ctf2JsonNumber number) + { + number = GetValue(json, property) as Ctf2JsonNumber; + return number != null; + } + + private static Ctf2JsonNumber GetRequiredNumber(Dictionary json, string property) + { + return TryGetNumber(json, property, out var number) + ? number + : throw new CtfMetadataException($"CTF 2 metadata is missing the required '{property}' number property."); + } + + private sealed class BuildContext + { + internal BuildContext(string clockName, bool prefixNames) + { + this.ClockName = clockName; + this.PrefixNames = prefixNames; + } + + internal string ClockName { get; } + + internal bool PrefixNames { get; } + + internal static BuildContext Header(string clockName) => new BuildContext(clockName, false); + + internal string MapName(string name) => this.PrefixNames ? "_" + name : name; + } + + private sealed class SelectorOption + { + internal SelectorOption(string name, IReadOnlyList> ranges) + { + this.Name = name; + this.Ranges = ranges; + } + + internal string Name { get; } + + internal IReadOnlyList> Ranges { get; } + } + } +} diff --git a/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs b/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs new file mode 100644 index 0000000..338770f --- /dev/null +++ b/CtfPlayback/Metadata/Ctf2/Ctf2VariableLengthIntegerDescriptor.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using CtfPlayback.FieldValues; +using CtfPlayback.Helpers; +using CtfPlayback.Metadata.Helpers; +using CtfPlayback.Metadata.TypeInterfaces; +using CtfPlayback.Metadata.Types; + +namespace CtfPlayback.Metadata.Ctf2 +{ + /// + /// CTF 2 variable-length (unsigned or signed LEB128) integer. + /// + internal sealed class Ctf2VariableLengthIntegerDescriptor + : CtfMetadataTypeDescriptor, + ICtfIntegerDescriptor + { + internal Ctf2VariableLengthIntegerDescriptor(bool signed, int displayBase, string map) + : base(CtfTypes.Integer) + { + this.Signed = signed; + this.Base = (short)displayBase; + this.Map = map; + } + + /// + public int Size => 64; + + /// + public bool Signed { get; } + + /// + public string Encoding => "none"; + + /// + public short Base { get; } + + /// + public string Map { get; } + + /// + public override int Align => 8; + + /// + public override CtfFieldValue Read(IPacketReader reader, CtfFieldValue parent = null) + { + Guard.NotNull(reader, nameof(reader)); + + reader.Align((uint)this.Align); + + ulong value = 0; + int shift = 0; + byte current; + do + { + current = reader.ReadBits(8)[0]; + if (shift < 64) + { + value |= (ulong)(current & 0x7f) << shift; + } + + shift += 7; + } + while ((current & 0x80) != 0); + + if (!this.Signed) + { + return new CtfIntegerValue(new IntegerLiteral(value), this); + } + + if (shift < 64 && (current & 0x40) != 0) + { + value |= ulong.MaxValue << shift; + } + + return new CtfIntegerValue(new IntegerLiteral((long)value), this); + } + + /// + public override string ToString() + { + return this.Signed ? "vlint" : "vluint"; + } + } +} diff --git a/CtfPlayback/Metadata/CtfMetadataExtensions.cs b/CtfPlayback/Metadata/CtfMetadataExtensions.cs new file mode 100644 index 0000000..9cbdecf --- /dev/null +++ b/CtfPlayback/Metadata/CtfMetadataExtensions.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using CtfPlayback.Metadata.Interfaces; + +namespace CtfPlayback.Metadata +{ + internal static class CtfMetadataExtensions + { + /// + /// Finds the stream descriptor with the given id. Stream ids are usually also their index, but + /// this is not required (e.g. CTF 2 metadata may declare streams in any order). + /// + internal static ICtfStreamDescriptor GetStream(this ICtfMetadata metadata, uint streamId) + { + var streams = metadata.Streams; + if (streamId < streams.Count && streams[(int)streamId].Id == streamId) + { + return streams[(int)streamId]; + } + + foreach (var stream in streams) + { + if (stream.Id == streamId) + { + return stream; + } + } + + throw new CtfPlaybackException($"The metadata does not describe stream id {streamId}."); + } + } +} diff --git a/CtfPlayback/Metadata/CtfMetadataText.cs b/CtfPlayback/Metadata/CtfMetadataText.cs new file mode 100644 index 0000000..435c52f --- /dev/null +++ b/CtfPlayback/Metadata/CtfMetadataText.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Text; + +namespace CtfPlayback.Metadata +{ + /// + /// Helpers for reading a metadata stream, which may be plain text or packetized (see https://diamon.org/ctf/#spec7.1). + /// + internal static class CtfMetadataText + { + private const uint PacketMagic = 0x75d11d57; + + // magic(4) + uuid(16) + checksum(4) + content_size(4) + packet_size(4) + 5 single-byte fields + private const int PacketHeaderSize = 37; + + internal static byte[] ReadAllBytes(Stream stream) + { + using (var memoryStream = new MemoryStream()) + { + stream.CopyTo(memoryStream); + return memoryStream.ToArray(); + } + } + + /// + /// Returns the metadata text, removing packet headers if the metadata is packetized. + /// + internal static string Read(byte[] metadata) + { + if (metadata.Length < PacketHeaderSize || BitConverter.ToUInt32(metadata, 0) != PacketMagic) + { + return Encoding.UTF8.GetString(metadata); + } + + var sb = new StringBuilder(); + int offset = 0; + while (offset + PacketHeaderSize <= metadata.Length) + { + if (BitConverter.ToUInt32(metadata, offset) != PacketMagic) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet magic number."); + } + + int contentBytes = (int)(BitConverter.ToUInt32(metadata, offset + 24) / 8); + int packetBytes = (int)(BitConverter.ToUInt32(metadata, offset + 28) / 8); + if (contentBytes < PacketHeaderSize || packetBytes < contentBytes || offset + contentBytes > metadata.Length) + { + throw new InvalidDataException("Metadata stream seems to be corrupt: invalid packet size."); + } + + sb.Append(Encoding.UTF8.GetString(metadata, offset + PacketHeaderSize, contentBytes - PacketHeaderSize)); + offset += packetBytes; + } + + return sb.ToString(); + } + } +} diff --git a/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs b/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs new file mode 100644 index 0000000..82207df --- /dev/null +++ b/CtfPlayback/Metadata/CtfVersionDetectingMetadataParser.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.IO; +using CtfPlayback.Metadata.AntlrParser; +using CtfPlayback.Metadata.Ctf2; +using CtfPlayback.Metadata.Interfaces; + +namespace CtfPlayback.Metadata +{ + /// + /// Parses either CTF 1.8 (TSDL) or CTF 2 (JSON) metadata, based on the content of the metadata stream. + /// + public sealed class CtfVersionDetectingMetadataParser + : ICtfMetadataParser + { + private readonly ICtfMetadataCustomization metadataCustomization; + private readonly ICtfMetadataBuilder metadataBuilder; + private readonly bool prefixCtf2EventFieldNamesWithUnderscore; + + /// + /// Constructor. + /// + /// Extension points for parsing metadata + /// The object used to build metadata + /// + /// See . + /// + public CtfVersionDetectingMetadataParser( + ICtfMetadataCustomization metadataCustomization, + ICtfMetadataBuilder metadataBuilder, + bool prefixCtf2EventFieldNamesWithUnderscore) + { + this.metadataCustomization = metadataCustomization; + this.metadataBuilder = metadataBuilder ?? throw new ArgumentNullException(nameof(metadataBuilder)); + this.prefixCtf2EventFieldNamesWithUnderscore = prefixCtf2EventFieldNamesWithUnderscore; + } + + /// + public ICtfMetadata Parse(Stream metadataStream) + { + if (metadataStream == null) + { + throw new ArgumentNullException(nameof(metadataStream)); + } + + byte[] metadata = CtfMetadataText.ReadAllBytes(metadataStream); + string text = CtfMetadataText.Read(metadata); + + if (Ctf2MetadataParser.IsCtf2Metadata(text)) + { + return new Ctf2MetadataParser(this.metadataBuilder, this.prefixCtf2EventFieldNamesWithUnderscore).Parse(text); + } + + using (var memoryStream = new MemoryStream(metadata, false)) + { + return new CtfAntlrMetadataParser(this.metadataCustomization, this.metadataBuilder).Parse(memoryStream); + } + } + } +} diff --git a/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs b/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs index b7f5540..ddbc514 100644 --- a/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs +++ b/CtfPlayback/Metadata/Types/CtfIntegerDescriptor.cs @@ -62,6 +62,9 @@ internal CtfIntegerDescriptor(CtfPropertyBag bag) internal string ByteOrder { get; private set; } + private bool IsExplicitlyBigEndian => + StringComparer.Ordinal.Equals(this.ByteOrder, "be") || StringComparer.Ordinal.Equals(this.ByteOrder, "network"); + /// public override string ToString() { @@ -98,6 +101,17 @@ internal CtfFieldValue Read(byte[] buffer, int byteCount) // if byte order is not set, or if the value is "native", then endianness is determined by the trace descriptor // if the byte order is "be" or "network", then it is big endian // if the byte order is "le", then it is little endian + // Only explicitly big endian, whole-byte integers are handled for now (e.g. IPv4 addresses in network order). + if (this.IsExplicitlyBigEndian && (this.Size % 8) == 0) + { + var littleEndianBuffer = new byte[byteCount]; + for (int x = 0; x < byteCount; x++) + { + littleEndianBuffer[x] = buffer[byteCount - 1 - x]; + } + + buffer = littleEndianBuffer; + } IntegerLiteral value; diff --git a/CtfUnitTest/Ctf2MetadataTests.cs b/CtfUnitTest/Ctf2MetadataTests.cs new file mode 100644 index 0000000..8e86ae3 --- /dev/null +++ b/CtfUnitTest/Ctf2MetadataTests.cs @@ -0,0 +1,281 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using CtfPlayback; +using CtfPlayback.FieldValues; +using CtfPlayback.Metadata; +using CtfPlayback.Metadata.Ctf2; +using CtfPlayback.Metadata.Interfaces; +using CtfPlayback.Metadata.InternalHelpers; +using CtfPlayback.Metadata.TypeInterfaces; +using CtfPlayback.Metadata.Types; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace CtfUnitTest +{ + [TestClass] + public class Ctf2MetadataTests + { + // Modeled on the metadata written by LTTng 2.16 for a kernel trace with perf counter contexts. + private static readonly string[] LTTngKernelFragments = + { + @"{""type"":""preamble"",""version"":2,""uuid"":[55,65,178,152,114,22,242,74,147,79,201,46,48,147,240,64]}", + @"{""type"":""field-class-alias"",""name"":""u32-erc-id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian"",""roles"":[""event-record-class-id""]}}", + @"{""type"":""field-class-alias"",""name"":""u64-ts"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian"",""roles"":[""default-clock-timestamp""]}}", + @"{""type"":""field-class-alias"",""name"":""er-header-compact"",""field-class"":{""type"":""structure"",""minimum-alignment"":8,""member-classes"":[{""name"":""id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":5,""byte-order"":""little-endian"",""mappings"":{""compact"":[[0,30]],""extended"":[[31,31]]},""roles"":[""event-record-class-id""]}},{""name"":""v"",""field-class"":{""type"":""variant"",""selector-field-location"":{""path"":[""id""]},""options"":[{""name"":""compact"",""selector-field-ranges"":[[0,30]],""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""timestamp"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":27,""byte-order"":""little-endian"",""roles"":[""default-clock-timestamp""]}}]}},{""name"":""extended"",""selector-field-ranges"":[[31,31]],""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""id"",""field-class"":""u32-erc-id""},{""name"":""timestamp"",""field-class"":""u64-ts""}]}}]}}]}}", + @"{""type"":""field-class-alias"",""name"":""pkt-ctx"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""timestamp_begin"",""field-class"":""u64-ts""},{""name"":""timestamp_end"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian"",""roles"":[""packet-end-default-clock-timestamp""]}},{""name"":""content_size"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""packet_size"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""events_discarded"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""cpu_id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}}]}}", + @"{""type"":""field-class-alias"",""name"":""pkt-header"",""field-class"":{""type"":""structure"",""member-classes"":[{""name"":""magic"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}},{""name"":""uuid"",""field-class"":{""type"":""static-length-blob"",""length"":16}},{""name"":""stream_id"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""alignment"":8,""byte-order"":""little-endian""}}]}}", + @"{""type"":""trace-class"",""uid"":""3741b298-7216-f24a-934f-c92e3093f040"",""environment"":{""domain"":""kernel"",""tracer_major"":2,""hostname"":""host""},""packet-header-field-class"":""pkt-header""}", + @"{""type"":""clock-class"",""id"":""monotonic"",""name"":""monotonic"",""frequency"":1000000000,""origin"":""unix-epoch"",""offset-from-origin"":{""seconds"":10,""cycles"":5}}", + @"{""type"":""data-stream-class"",""id"":1,""default-clock-class-id"":""monotonic"",""packet-context-field-class"":""pkt-ctx"",""event-record-header-field-class"":""er-header-compact"",""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}}]}}", + @"{""type"":""data-stream-class"",""id"":0,""default-clock-class-id"":""monotonic"",""packet-context-field-class"":""pkt-ctx"",""event-record-header-field-class"":""er-header-compact"",""event-record-common-context-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""perf_cpu_cycles"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""_callstack_kernel_length"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""callstack_kernel"",""field-class"":{""type"":""dynamic-length-array"",""element-field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":64,""byte-order"":""little-endian"",""alignment"":8,""preferred-display-base"":16},""length-field-location"":{""path"":[""_callstack_kernel_length""]}}}]}}", + @"{""type"":""event-record-class"",""data-stream-class-id"":1,""id"":0,""name"":""sched_wakeup"",""payload-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""comm"",""field-class"":{""type"":""static-length-string"",""length"":16}},{""name"":""tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}}]}}", + @"{""type"":""event-record-class"",""data-stream-class-id"":0,""id"":0,""name"":""sched_switch"",""attributes"":{""lttng.org,2009"":{""log-level"":13}},""payload-field-class"":{""type"":""structure"",""member-classes"":[{""name"":""prev_tid"",""field-class"":{""type"":""fixed-length-signed-integer"",""length"":32,""byte-order"":""little-endian"",""alignment"":8}},{""name"":""filename"",""field-class"":{""type"":""null-terminated-string""}},{""name"":""address_ipv4"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":32,""byte-order"":""big-endian"",""alignment"":8,""preferred-display-base"":16}},{""name"":""ihl"",""field-class"":{""type"":""fixed-length-unsigned-integer"",""length"":4,""byte-order"":""big-endian""}},{""name"":""ratio"",""field-class"":{""type"":""fixed-length-floating-point-number"",""length"":64,""byte-order"":""little-endian"",""alignment"":8}}]}}", + }; + + private static string LTTngKernelMetadata => string.Concat(LTTngKernelFragments.Select(fragment => "\u001e" + fragment + "\n")); + + [TestMethod] + public void DetectsCtf2Metadata() + { + Assert.IsTrue(Ctf2MetadataParser.IsCtf2Metadata(LTTngKernelMetadata)); + Assert.IsTrue(Ctf2MetadataParser.IsCtf2Metadata("\n \u001e{}")); + Assert.IsFalse(Ctf2MetadataParser.IsCtf2Metadata("/* CTF 1.8 */ trace { major = 1; };")); + Assert.IsFalse(Ctf2MetadataParser.IsCtf2Metadata(string.Empty)); + } + + [TestMethod] + public void ParsesTraceClockAndStreams() + { + var builder = Parse(LTTngKernelMetadata); + + Assert.AreEqual(Guid.Parse("3741b298-7216-f24a-934f-c92e3093f040"), builder.TraceDescriptor.Uuid); + Assert.AreEqual("kernel", builder.EnvironmentDescriptor.Properties["domain"]); + Assert.AreEqual("2", builder.EnvironmentDescriptor.Properties["tracer_major"]); + CollectionAssert.AreEqual( + new[] { "magic", "uuid", "stream_id" }, + builder.TraceDescriptor.PacketHeader.Fields.Select(f => f.Name).ToArray()); + + var clock = builder.ClocksByName["monotonic"]; + Assert.AreEqual(1000000000ul, clock.Frequency); + Assert.AreEqual(10000000005ul, clock.Offset); + + // Streams are declared out of order in the metadata but are stored by id. + CollectionAssert.AreEqual(new uint[] { 0, 1 }, builder.Streams.Select(s => s.Id).ToArray()); + } + + [TestMethod] + public void EventHeaderVariantSelectorIsAnEnumeration() + { + var builder = Parse(LTTngKernelMetadata); + var header = builder.Streams[0].EventHeader; + + var id = header.GetField("id").TypeDescriptor as ICtfEnumDescriptor; + Assert.IsNotNull(id); + Assert.AreEqual("compact", id.GetName(5ul)); + Assert.AreEqual("extended", id.GetName(31ul)); + + var variant = header.GetField("v").TypeDescriptor as ICtfVariantDescriptor; + Assert.IsNotNull(variant); + Assert.AreEqual("id", variant.Switch); + + var compactTimestamp = ((ICtfStructDescriptor)variant.GetVariant("compact").TypeDescriptor).GetField("timestamp").TypeDescriptor as ICtfIntegerDescriptor; + Assert.AreEqual(27, compactTimestamp.Size); + Assert.AreEqual("clock.monotonic.value", compactTimestamp.Map); + } + + [TestMethod] + public void EventFieldNamesArePrefixedLikeCtf18() + { + var builder = Parse(LTTngKernelMetadata); + + var context = builder.Streams[0].EventContext; + CollectionAssert.AreEqual( + new[] { "_perf_cpu_cycles", "__callstack_kernel_length", "_callstack_kernel" }, + context.Fields.Select(f => f.Name).ToArray()); + + var callstack = context.GetField("_callstack_kernel").TypeDescriptor as ICtfArrayDescriptor; + Assert.AreEqual("__callstack_kernel_length", callstack.Index); + + // Header and packet scopes keep their names. + Assert.IsNotNull(builder.Streams[0].PacketContext.GetField("timestamp_begin")); + } + + [TestMethod] + public void EventsAreIdentifiedByStreamAndId() + { + var builder = Parse(LTTngKernelMetadata); + + Assert.AreEqual(2, builder.AddedEvents.Count); + + var sched_switch = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_switch"); + Assert.AreEqual("0", sched_switch.Assignments["stream_id"]); + Assert.AreEqual("0", sched_switch.Assignments["id"]); + Assert.AreEqual("13", sched_switch.Assignments["loglevel"]); + + var sched_wakeup = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_wakeup"); + Assert.AreEqual("1", sched_wakeup.Assignments["stream_id"]); + Assert.AreEqual("0", sched_wakeup.Assignments["id"]); + + var payload = (ICtfStructDescriptor)sched_wakeup.TypeDeclarations["fields"]; + var comm = payload.GetField("_comm").TypeDescriptor as ICtfArrayDescriptor; + Assert.IsNotNull(comm); + Assert.AreEqual("16", comm.Index); + Assert.AreEqual("UTF8", ((ICtfIntegerDescriptor)comm.Type).Encoding); + + var switchPayload = (ICtfStructDescriptor)sched_switch.TypeDeclarations["fields"]; + Assert.IsInstanceOfType(switchPayload.GetField("_filename").TypeDescriptor, typeof(ICtfStringDescriptor)); + Assert.AreEqual(4, ((ICtfIntegerDescriptor)switchPayload.GetField("_ihl").TypeDescriptor).Size); + + var ratio = switchPayload.GetField("_ratio").TypeDescriptor as ICtfFloatingPointDescriptor; + Assert.IsNotNull(ratio); + Assert.AreEqual(11, ratio.Exponent); + Assert.AreEqual(53, ratio.Mantissa); + } + + [TestMethod] + public void VersionDetectingParserHandlesPacketizedCtf2Metadata() + { + byte[] text = Encoding.UTF8.GetBytes(LTTngKernelMetadata); + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + writer.Write(0x75d11d57u); + writer.Write(new byte[16]); + writer.Write(0u); + writer.Write((uint)((37 + text.Length) * 8)); + writer.Write((uint)((37 + text.Length) * 8)); + writer.Write(new byte[] { 0, 0, 0, 2, 0 }); + writer.Write(text); + stream.Position = 0; + + var builder = new Ctf2TestMetadataBuilder(); + new CtfVersionDetectingMetadataParser(null, builder, true).Parse(stream); + + Assert.AreEqual(2, builder.Streams.Count); + Assert.AreEqual(2, builder.AddedEvents.Count); + } + + [TestMethod] + public void RejectsUnsupportedVersion() + { + Assert.ThrowsException(() => Parse("\u001e{\"type\":\"preamble\",\"version\":3}")); + } + + [TestMethod] + public void ReadsBigEndianIntegers() + { + var builder = Parse(LTTngKernelMetadata); + var sched_switch = builder.AddedEvents.Single(e => e.Assignments["name"] == "sched_switch"); + var address = (CtfIntegerDescriptor)((ICtfStructDescriptor)sched_switch.TypeDeclarations["fields"]).GetField("_address_ipv4").TypeDescriptor; + + var value = (CtfIntegerValue)address.Read(new byte[] { 127, 0, 0, 1 }, 4); + Assert.AreEqual(0x7F000001ul, value.Value.ValueAsUlong); + } + + [TestMethod] + public void ReadsVariableLengthIntegers() + { + var unsigned = new Ctf2VariableLengthIntegerDescriptor(false, 10, null); + var signed = new Ctf2VariableLengthIntegerDescriptor(true, 10, null); + + Assert.AreEqual(624485ul, ((CtfIntegerValue)unsigned.Read(new BytePacketReader(0xE5, 0x8E, 0x26))).Value.ValueAsUlong); + Assert.AreEqual(-123456L, ((CtfIntegerValue)signed.Read(new BytePacketReader(0xC0, 0xBB, 0x78))).Value.ValueAsLong); + } + + [TestMethod] + public void JsonReaderHandlesLargeNumbersAndEscapes() + { + var json = (Dictionary)Ctf2Json.Parse(@"{ ""a"": 18446744073709551615, ""b"": ""x\""\u0041\n"", ""c"": [true, false, null, -5] }"); + + Assert.AreEqual(ulong.MaxValue, ((Ctf2JsonNumber)json["a"]).AsULong()); + Assert.AreEqual("x\"A\n", json["b"]); + var array = (List)json["c"]; + Assert.AreEqual(true, array[0]); + Assert.AreEqual(false, array[1]); + Assert.IsNull(array[2]); + Assert.AreEqual(-5L, ((Ctf2JsonNumber)array[3]).AsLong()); + + Assert.ThrowsException(() => Ctf2Json.Parse("{\"a\": }")); + } + + private static Ctf2TestMetadataBuilder Parse(string metadata) + { + var builder = new Ctf2TestMetadataBuilder(); + new Ctf2MetadataParser(builder, prefixEventFieldNamesWithUnderscore: true).Parse(metadata); + return builder; + } + + private sealed class AddedEvent + { + public IReadOnlyDictionary Assignments { get; set; } + + public IReadOnlyDictionary TypeDeclarations { get; set; } + } + + private sealed class Ctf2TestMetadataBuilder + : ICtfMetadataBuilder + { + private readonly List clocks = new List(); + private readonly List streams = new List(); + + public List AddedEvents { get; } = new List(); + + public ICtfTraceDescriptor TraceDescriptor { get; private set; } + + public ICtfEnvironmentDescriptor EnvironmentDescriptor { get; private set; } + + public IReadOnlyList Clocks => this.clocks; + + public IReadOnlyDictionary ClocksByName => this.clocks.ToDictionary(c => c.Name); + + public IReadOnlyList Streams => this.streams; + + public IReadOnlyList Events => Array.Empty(); + + public void SetTraceDescriptor(ICtfTraceDescriptor traceDescriptor) => this.TraceDescriptor = traceDescriptor; + + public void SetEnvironmentDescriptor(ICtfEnvironmentDescriptor environmentDescriptor) => this.EnvironmentDescriptor = environmentDescriptor; + + public void AddEvent(IReadOnlyDictionary assignments, IReadOnlyDictionary typeDeclarations) + { + this.AddedEvents.Add(new AddedEvent { Assignments = assignments, TypeDeclarations = typeDeclarations }); + } + + public void AddClock(ICtfClockDescriptor clockDescriptor) => this.clocks.Add(clockDescriptor); + + public void AddStream(ICtfStreamDescriptor streamDescriptor) => this.streams.Add(streamDescriptor); + } + + private sealed class BytePacketReader + : IPacketReader + { + private readonly Queue bytes; + + public BytePacketReader(params byte[] bytes) + { + this.bytes = new Queue(bytes); + } + + public bool EndOfStream => this.bytes.Count == 0; + + public uint RemainingBufferedBitCount => (uint)this.bytes.Count * 8; + + public byte[] ReadBits(uint bitCount) + { + Assert.AreEqual(8u, bitCount); + return new[] { this.bytes.Dequeue() }; + } + + public byte[] ReadString() => throw new NotSupportedException(); + + public void Align(uint bitCount) + { + } + } + } +} diff --git a/LTTngCds/CookerData/LTTngEvent.cs b/LTTngCds/CookerData/LTTngEvent.cs index cb8f5ad..e3de897 100644 --- a/LTTngCds/CookerData/LTTngEvent.cs +++ b/LTTngCds/CookerData/LTTngEvent.cs @@ -50,10 +50,15 @@ internal LTTngEvent(LTTngEvent lttngEvent, long timestampAdjustmentNano) : this( } /// - /// Event Id + /// Event Id. Event ids are only unique within a stream, see . /// public uint Id => this.eventDescriptor.Id; + /// + /// Id of the stream (LTTng channel) that contains the event. + /// + public uint StreamId => this.ctfEvent.StreamId; + /// /// Event name /// diff --git a/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs b/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs index 4c7d37f..d7f8923 100644 --- a/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs +++ b/LTTngCds/CtfExtensions/Descriptors/EventDescriptor.cs @@ -31,6 +31,11 @@ internal EventDescriptor( this.Name = string.Intern(name.Trim(new[] { '"' })); this.SetPayload(typeDeclarations); + + if (typeDeclarations.TryGetValue("context", out var context)) + { + this.Context = context; + } } public uint Id { get; private set; } @@ -41,9 +46,9 @@ internal EventDescriptor( public uint LogLevel { get; private set; } - // LTTng defines no context for an event + // CTF 1.8 LTTng traces define no context for an event; CTF 2 traces may define a specific context. - public ICtfTypeDescriptor Context => null; + public ICtfTypeDescriptor Context { get; } public ICtfTypeDescriptor Payload { get; private set; } diff --git a/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs b/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs index 5e7ddbb..576d920 100644 --- a/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs +++ b/LTTngCds/CtfExtensions/FolderInput/LTTngFolderInput.cs @@ -35,7 +35,7 @@ public LTTngFolderInput(string folderPath) Debug.Assert(traceDirectoryPath != null, nameof(traceDirectoryPath) + " != null"); var associatedEntries = Directory.GetFiles(traceDirectoryPath).Where(entry => - Path.GetFileName(entry).StartsWith("chan")); + LTTngStreamFiles.IsEventStreamFile(Path.GetFileName(entry))); traceInput.EventStreams = associatedEntries.Select( fileName => new LTTngFileInputStream(fileName)).Cast().ToList(); diff --git a/LTTngCds/CtfExtensions/LTTngMetadata.cs b/LTTngCds/CtfExtensions/LTTngMetadata.cs index 49ffa4d..78bd29b 100644 --- a/LTTngCds/CtfExtensions/LTTngMetadata.cs +++ b/LTTngCds/CtfExtensions/LTTngMetadata.cs @@ -15,7 +15,7 @@ internal class LTTngMetadata private readonly List streams = new List(); private readonly List clocks = new List(); private readonly List events = new List(); - private readonly Dictionary eventsById = new Dictionary(); + private readonly Dictionary<(int StreamId, uint EventId), ICtfEventDescriptor> eventsByStreamAndId = new Dictionary<(int StreamId, uint EventId), ICtfEventDescriptor>(); private readonly Dictionary clocksByName = new Dictionary(); public ICtfTraceDescriptor TraceDescriptor { get; private set; } @@ -30,7 +30,13 @@ internal class LTTngMetadata public IReadOnlyList Events => this.events; - public IReadOnlyDictionary EventByEventId => this.eventsById; + /// + /// Event ids are only unique within a stream (e.g. with multiple channels, or in CTF 2 traces). + /// + public bool TryGetEvent(uint streamId, uint eventId, out ICtfEventDescriptor eventDescriptor) + { + return this.eventsByStreamAndId.TryGetValue(((int)streamId, eventId), out eventDescriptor); + } public void SetTraceDescriptor(ICtfTraceDescriptor traceDescriptor) { @@ -52,7 +58,7 @@ public void AddEvent( { var eventDescriptor = new EventDescriptor(assignments, typeDeclarations); this.events.Add(eventDescriptor); - this.eventsById.Add(eventDescriptor.Id, eventDescriptor); + this.eventsByStreamAndId.Add((eventDescriptor.Stream, eventDescriptor.Id), eventDescriptor); } public void AddClock(ICtfClockDescriptor clockDescriptor) diff --git a/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs b/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs index 0e50e8b..2628802 100644 --- a/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs +++ b/LTTngCds/CtfExtensions/LTTngPlaybackCustomization.cs @@ -10,7 +10,6 @@ using CtfPlayback.FieldValues; using CtfPlayback.Inputs; using CtfPlayback.Metadata; -using CtfPlayback.Metadata.AntlrParser; using CtfPlayback.Metadata.Interfaces; using LTTngCds.CookerData; using LTTngCds.CtfExtensions.Descriptors; @@ -41,7 +40,10 @@ public virtual ICtfMetadataParser CreateMetadataParser( ICtfTraceInput traceInput) { this.metadataCustomization.PrepareForNewTrace(traceInput.MetadataStream); - return new CtfAntlrMetadataParser(this.metadataCustomization, this.metadataCustomization.LTTngMetadata); + return new CtfVersionDetectingMetadataParser( + this.metadataCustomization, + this.metadataCustomization.LTTngMetadata, + prefixCtf2EventFieldNamesWithUnderscore: true); } public bool GetTimestampsFromPacketContext( @@ -223,22 +225,24 @@ public ulong GetPacketContentBitCount(ICtfPacket ctfPacket) public ICtfEventDescriptor GetEventDescriptor(ICtfEvent ctfEvent, ICtfMetadata metadata) { uint id = this.GetEventId(ctfEvent); + uint streamId = ctfEvent.StreamId; ICtfEventDescriptor eventDescriptor; if (metadata is LTTngMetadata typedMetadata) { // optimization if we got our own typed metadata back // we check for success below - typedMetadata.EventByEventId.TryGetValue(id, out eventDescriptor); + typedMetadata.TryGetEvent(streamId, id, out eventDescriptor); } else { - eventDescriptor = metadata.Events.FirstOrDefault(x => x.Id == id); + eventDescriptor = metadata.Events.FirstOrDefault(x => + x.Id == id && (!(x is EventDescriptor lttngDescriptor) || lttngDescriptor.Stream == streamId)); } if (eventDescriptor == null) { - throw new LTTngPlaybackException($"Unable to find event descriptor for event id={id}."); + throw new LTTngPlaybackException($"Unable to find event descriptor for stream id={streamId}, event id={id}."); } return eventDescriptor; diff --git a/LTTngCds/CtfExtensions/LTTngStreamFiles.cs b/LTTngCds/CtfExtensions/LTTngStreamFiles.cs new file mode 100644 index 0000000..80efbb2 --- /dev/null +++ b/LTTngCds/CtfExtensions/LTTngStreamFiles.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; + +namespace LTTngCds.CtfExtensions +{ + internal static class LTTngStreamFiles + { + /// + /// LTTng names event stream files "<channel name>_<cpu>", e.g. "channel0_3" or "sw-chan_3". + /// + /// File name without directory + /// true if the file is an event stream + internal static bool IsEventStreamFile(string fileName) + { + if (string.IsNullOrEmpty(fileName) || + fileName.StartsWith(".", StringComparison.Ordinal) || + StringComparer.Ordinal.Equals(fileName, "metadata")) + { + return false; + } + + int separatorIndex = fileName.LastIndexOf('_'); + if (separatorIndex <= 0 || separatorIndex == fileName.Length - 1) + { + return false; + } + + for (int index = separatorIndex + 1; index < fileName.Length; index++) + { + if (!char.IsDigit(fileName[index])) + { + return false; + } + } + + return true; + } + } +} diff --git a/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs b/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs index b843252..c017780 100644 --- a/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs +++ b/LTTngCds/CtfExtensions/ZipArchiveInput/LTTngZipArchiveInput.cs @@ -39,7 +39,7 @@ public LTTngZipArchiveInput(ZipArchive archive) var associatedArchiveEntries = archive.Entries.Where(entry => Path.GetDirectoryName(entry.FullName) == traceDirectoryPath && - Path.GetFileName(entry.FullName).StartsWith("chan")); + LTTngStreamFiles.IsEventStreamFile(Path.GetFileName(entry.FullName))); traceInput.EventStreams = associatedArchiveEntries.Select( archiveEntry => new LTTngZipArchiveInputStream(archiveEntry)).Cast().ToList(); diff --git a/LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs b/LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs new file mode 100644 index 0000000..100cb5a --- /dev/null +++ b/LTTngDataExtUnitTest/LTTngCtf2UnitTest.cs @@ -0,0 +1,221 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using LTTngDataExtensions.DataOutputTypes; +using LTTngDataExtensions.SourceDataCookers; +using LTTngDataExtensions.SourceDataCookers.Thread; +using LTTngDataExtensions.Tables; +using Microsoft.Performance.SDK.Extensibility; +using Microsoft.Performance.SDK.Processing; +using Microsoft.Performance.Toolkit.Engine; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace LTTngDataExtUnitTest +{ + /// + /// Processes a small synthetic LTTng kernel trace written in the CTF 2 format, with channels that are not named + /// "chan*" and event ids that are only unique within a channel (as written by LTTng 2.15+). + /// + [TestClass] + public class LTTngCtf2UnitTest + { + private const string Metadata = + "\u001e{\"type\":\"preamble\",\"version\":2}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u64-ts\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u64\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"u32\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":32,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"s32\",\"field-class\":{\"type\":\"fixed-length-signed-integer\",\"length\":32,\"alignment\":8,\"byte-order\":\"little-endian\"}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"comm\",\"field-class\":{\"type\":\"static-length-string\",\"length\":16}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"er-header-compact\",\"field-class\":{\"type\":\"structure\",\"minimum-alignment\":8,\"member-classes\":[{\"name\":\"id\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":5,\"byte-order\":\"little-endian\",\"mappings\":{\"compact\":[[0,30]],\"extended\":[[31,31]]}}},{\"name\":\"v\",\"field-class\":{\"type\":\"variant\",\"selector-field-location\":{\"path\":[\"id\"]},\"options\":[{\"name\":\"compact\",\"selector-field-ranges\":[[0,30]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":27,\"byte-order\":\"little-endian\",\"roles\":[\"default-clock-timestamp\"]}}]}},{\"name\":\"extended\",\"selector-field-ranges\":[[31,31]],\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"id\",\"field-class\":\"u32\"},{\"name\":\"timestamp\",\"field-class\":\"u64-ts\"}]}}]}}]}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"pkt-ctx\",\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"timestamp_begin\",\"field-class\":\"u64-ts\"},{\"name\":\"timestamp_end\",\"field-class\":{\"type\":\"fixed-length-unsigned-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\",\"roles\":[\"packet-end-default-clock-timestamp\"]}},{\"name\":\"content_size\",\"field-class\":\"u64\"},{\"name\":\"packet_size\",\"field-class\":\"u64\"},{\"name\":\"events_discarded\",\"field-class\":\"u64\"},{\"name\":\"cpu_id\",\"field-class\":\"u32\"}]}}\n" + + "\u001e{\"type\":\"field-class-alias\",\"name\":\"pkt-header\",\"field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"magic\",\"field-class\":\"u32\"},{\"name\":\"uuid\",\"field-class\":{\"type\":\"static-length-blob\",\"length\":16}},{\"name\":\"stream_id\",\"field-class\":\"u32\"}]}}\n" + + "\u001e{\"type\":\"trace-class\",\"environment\":{\"domain\":\"kernel\",\"tracer_name\":\"lttng-modules\",\"tracer_major\":2,\"tracer_minor\":16},\"packet-header-field-class\":\"pkt-header\"}\n" + + "\u001e{\"type\":\"clock-class\",\"id\":\"monotonic\",\"name\":\"monotonic\",\"frequency\":1000000000,\"origin\":\"unix-epoch\",\"offset-from-origin\":{\"seconds\":1790000000,\"cycles\":0}}\n" + + "\u001e{\"type\":\"data-stream-class\",\"id\":1,\"default-clock-class-id\":\"monotonic\",\"packet-context-field-class\":\"pkt-ctx\",\"event-record-header-field-class\":\"er-header-compact\",\"event-record-common-context-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"pid\",\"field-class\":\"s32\"},{\"name\":\"tid\",\"field-class\":\"s32\"},{\"name\":\"procname\",\"field-class\":\"comm\"},{\"name\":\"prio\",\"field-class\":\"s32\"}]}}\n" + + "\u001e{\"type\":\"data-stream-class\",\"id\":0,\"default-clock-class-id\":\"monotonic\",\"packet-context-field-class\":\"pkt-ctx\",\"event-record-header-field-class\":\"er-header-compact\",\"event-record-common-context-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"perf_cpu_cycles\",\"field-class\":\"u64\"},{\"name\":\"perf_cpu_instructions\",\"field-class\":\"u64\"},{\"name\":\"pid\",\"field-class\":\"s32\"},{\"name\":\"tid\",\"field-class\":\"s32\"},{\"name\":\"procname\",\"field-class\":\"comm\"},{\"name\":\"prio\",\"field-class\":\"s32\"}]}}\n" + + "\u001e{\"type\":\"event-record-class\",\"data-stream-class-id\":1,\"id\":0,\"name\":\"sched_wakeup\",\"payload-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"comm\",\"field-class\":\"comm\"},{\"name\":\"tid\",\"field-class\":\"s32\"},{\"name\":\"prio\",\"field-class\":\"s32\"},{\"name\":\"target_cpu\",\"field-class\":\"s32\"}]}}\n" + + "\u001e{\"type\":\"event-record-class\",\"data-stream-class-id\":0,\"id\":0,\"name\":\"sched_switch\",\"payload-field-class\":{\"type\":\"structure\",\"member-classes\":[{\"name\":\"prev_comm\",\"field-class\":\"comm\"},{\"name\":\"prev_tid\",\"field-class\":\"s32\"},{\"name\":\"prev_prio\",\"field-class\":\"s32\"},{\"name\":\"prev_state\",\"field-class\":{\"type\":\"fixed-length-signed-integer\",\"length\":64,\"alignment\":8,\"byte-order\":\"little-endian\"}},{\"name\":\"next_comm\",\"field-class\":\"comm\"},{\"name\":\"next_tid\",\"field-class\":\"s32\"},{\"name\":\"next_prio\",\"field-class\":\"s32\"}]}}\n"; + + private static string tracePath; + private static RuntimeExecutionResults results; + private static DataCookerPath threadCookerPath; + private static DataCookerPath genericCookerPath; + + [ClassInitialize] + public static void ProcessTrace(TestContext context) + { + tracePath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ctf"); + WriteTrace(tracePath); + + var runtime = Engine.Create(new FileDataSource(tracePath)); + + var threadCooker = new LTTngThreadDataCooker(); + threadCookerPath = threadCooker.Path; + runtime.EnableCooker(threadCookerPath); + + var genericCooker = new LTTngGenericEventDataCooker(); + genericCookerPath = genericCooker.Path; + runtime.EnableCooker(genericCookerPath); + + runtime.EnableTable(ExecutionEventTable.TableDescriptor); + + results = runtime.Process(); + } + + [ClassCleanup] + public static void Cleanup() + { + if (tracePath != null && File.Exists(tracePath)) + { + File.Delete(tracePath); + } + } + + [TestMethod] + public void ProcessesAllChannels() + { + Assert.AreEqual(0, results.ProcessingErrors.Count(), string.Join(Environment.NewLine, results.ProcessingErrors.Select(e => e.ProcessFault?.ToString()))); + + var events = results.QueryOutput>( + new DataOutputPath(genericCookerPath, nameof(LTTngGenericEventDataCooker.Events))); + + var names = Enumerable.Range(0, (int)events.Count).Select(i => events[i].EventName).ToList(); + Assert.AreEqual(3, names.Count(n => n == "sched_switch")); + Assert.AreEqual(1, names.Count(n => n == "sched_wakeup")); + + var wakeup = Enumerable.Range(0, (int)events.Count).Select(i => events[i]).Single(e => e.EventName == "sched_wakeup"); + Assert.AreEqual("taskA", wakeup.FieldValues[wakeup.FieldNames.IndexOf("_comm")]); + } + + [TestMethod] + public void ExecutionEventsHavePerformanceCounterDeltas() + { + var executionEvents = results.QueryOutput>( + new DataOutputPath(threadCookerPath, nameof(LTTngThreadDataCooker.ExecutionEvents))); + + Assert.AreEqual(2, executionEvents.Count); + + var taskA = executionEvents.Single(e => e.NextTid == 100); + Assert.AreEqual("taskA", taskA.NextImage); + Assert.AreEqual(500, taskA.PerformanceCountersDiffByName["_perf_cpu_cycles"]); + Assert.AreEqual(600, taskA.PerformanceCountersDiffByName["_perf_cpu_instructions"]); + + var idle = executionEvents.Single(e => e.NextTid == 0); + Assert.AreEqual(300, idle.PerformanceCountersDiffByName["_perf_cpu_cycles"]); + Assert.AreEqual(400, idle.PerformanceCountersDiffByName["_perf_cpu_instructions"]); + + var table = results.BuildTable(ExecutionEventTable.TableDescriptor); + var cycles = table.Columns.Single(c => c.Configuration.Metadata.Name == "CPU Cycle"); + var instructions = table.Columns.Single(c => c.Configuration.Metadata.Name == "Instruction Count"); + CollectionAssert.AreEquivalent(new long[] { 500, 300 }, Enumerable.Range(0, table.RowCount).Select(i => (long)cycles.Project(i)).ToArray()); + CollectionAssert.AreEquivalent(new long[] { 600, 400 }, Enumerable.Range(0, table.RowCount).Select(i => (long)instructions.Project(i)).ToArray()); + } + + private static void WriteTrace(string path) + { + using (var archive = ZipFile.Open(path, ZipArchiveMode.Create)) + { + WriteEntry(archive, "trace/kernel/metadata", Encoding.UTF8.GetBytes(Metadata)); + + // Channel "sw-chan" (stream class 0) on CPU 0: idle -> taskA -> idle -> taskA. + WriteEntry(archive, "trace/kernel/sw-chan_0", Packet(0, 1000, 3000, + SchedSwitch(1000, cycles: 1000, instructions: 2000, prevTid: 0, prevComm: "swapper/0", nextTid: 100, nextComm: "taskA"), + SchedSwitch(2000, cycles: 1500, instructions: 2600, prevTid: 100, prevComm: "taskA", nextTid: 0, nextComm: "swapper/0"), + SchedSwitch(3000, cycles: 1800, instructions: 3000, prevTid: 0, prevComm: "swapper/0", nextTid: 100, nextComm: "taskA"))); + + // Channel "other-chan" (stream class 1) on CPU 0: its event id 0 is sched_wakeup, not sched_switch. + WriteEntry(archive, "trace/kernel/other-chan_0", Packet(1, 1500, 1500, SchedWakeup(1500, tid: 100, comm: "taskA"))); + } + } + + private static void WriteEntry(ZipArchive archive, string name, byte[] content) + { + using (var stream = archive.CreateEntry(name).Open()) + { + stream.Write(content, 0, content.Length); + } + } + + private static byte[] Packet(uint streamId, ulong begin, ulong end, params byte[][] events) + { + const int headerAndContextSize = 4 + 16 + 4 + (5 * 8) + 4; + ulong sizeInBits = (ulong)(headerAndContextSize + events.Sum(e => e.Length)) * 8; + + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + writer.Write(0xC1FC1FC1u); + writer.Write(new byte[16]); + writer.Write(streamId); + writer.Write(begin); + writer.Write(end); + writer.Write(sizeInBits); + writer.Write(sizeInBits); + writer.Write(0ul); + writer.Write(0u); + foreach (var e in events) + { + writer.Write(e); + } + + return stream.ToArray(); + } + + private static byte[] SchedSwitch(uint timestamp, ulong cycles, ulong instructions, int prevTid, string prevComm, int nextTid, string nextComm) + { + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + WriteCompactHeader(writer, 0, timestamp); + writer.Write(cycles); + writer.Write(instructions); + WriteProcessContext(writer, prevTid, prevComm); + writer.Write(Comm(prevComm)); + writer.Write(prevTid); + writer.Write(20); + writer.Write(0L); + writer.Write(Comm(nextComm)); + writer.Write(nextTid); + writer.Write(20); + return stream.ToArray(); + } + + private static byte[] SchedWakeup(uint timestamp, int tid, string comm) + { + var stream = new MemoryStream(); + var writer = new BinaryWriter(stream); + WriteCompactHeader(writer, 0, timestamp); + WriteProcessContext(writer, 0, "swapper/0"); + writer.Write(Comm(comm)); + writer.Write(tid); + writer.Write(20); + writer.Write(0); + return stream.ToArray(); + } + + private static void WriteCompactHeader(BinaryWriter writer, uint id, uint timestamp) + { + // 5-bit event id followed by a 27-bit timestamp, packed least significant bit first. + writer.Write(id | (timestamp << 5)); + } + + private static void WriteProcessContext(BinaryWriter writer, int tid, string procname) + { + writer.Write(tid); + writer.Write(tid); + writer.Write(Comm(procname)); + writer.Write(20); + } + + private static byte[] Comm(string value) + { + var bytes = new byte[16]; + Encoding.UTF8.GetBytes(value, 0, value.Length, bytes, 0); + return bytes; + } + } +} diff --git a/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs b/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs index 3b46c1d..319414d 100644 --- a/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs +++ b/LTTngDataExtensions/DataOutputTypes/LTTngGenericEvent.cs @@ -16,11 +16,13 @@ public class EventKind private class Key { private string domain; + private uint streamId; private uint id; - public Key(string domain, uint id) + public Key(string domain, uint streamId, uint id) { this.domain = domain ?? string.Empty; + this.streamId = streamId; this.id = id; } @@ -28,7 +30,7 @@ public override bool Equals(object obj) { if (obj is Key key) { - return domain.Equals(key.domain) && id.Equals(key.id); + return domain.Equals(key.domain) && streamId.Equals(key.streamId) && id.Equals(key.id); } else { @@ -40,7 +42,8 @@ public override int GetHashCode() { int h1 = domain.GetHashCode(); int h2 = id.GetHashCode(); - return ((h1 << 5) + h1) ^ h2; + int h3 = streamId.GetHashCode(); + return ((((h1 << 5) + h1) ^ h2) * 31) ^ h3; } } @@ -76,13 +79,26 @@ public EventKind(LTTngContext context, uint id, string name, IReadOnlyList + /// Event ids are only unique within a stream (LTTng channel), so kinds are registered per stream. + /// + public static bool TryGetRegisteredKind(string domain, uint streamId, uint id, out EventKind kind) + { + return RegisteredKinds.TryGetValue(new Key(domain, streamId, id), out kind); } public static EventKind RegisterKind(LTTngContext context, uint id, string name, IReadOnlyList fields) + { + return RegisterKind(context, 0, id, name, fields); + } + + public static EventKind RegisterKind(LTTngContext context, uint streamId, uint id, string name, IReadOnlyList fields) { EventKind kind = new EventKind(context, id, name, fields); - RegisteredKinds.Add(new Key(context.Domain, id), kind); + RegisteredKinds.Add(new Key(context.Domain, streamId, id), kind); return kind; } } @@ -103,9 +119,9 @@ public LTTngGenericEvent(LTTngEvent data, LTTngContext context) this.CpuId = context.CurrentCpu; this.DiscardedEvents = data.DiscardedEvents; - if (!EventKind.TryGetRegisteredKind(context.Domain, data.Id, out this.kind)) + if (!EventKind.TryGetRegisteredKind(context.Domain, data.StreamId, data.Id, out this.kind)) { - this.kind = EventKind.RegisterKind(context, data.Id, data.Name, payload.Fields); + this.kind = EventKind.RegisterKind(context, data.StreamId, data.Id, data.Name, payload.Fields); } // As this is being written, all columns are of type 'T', so all rows are the same. For generic events, diff --git a/LinuxTraceLogCapture.md b/LinuxTraceLogCapture.md index 5371774..fdf5df0 100644 --- a/LinuxTraceLogCapture.md +++ b/LinuxTraceLogCapture.md @@ -44,7 +44,9 @@ Supports: - Block IO / Disk Activity - Diagnostic Messages -Once you have everything set up you just need to decide what kind of information you are looking for and begin tracing. +Traces in both the CTF 1.8 and CTF 2 formats are supported (CTF 2 is the default trace format since LTTng 2.15). + +Once you have everything set up you just need to decide what kind of information you are looking for and begin tracing. In this example we are looking at process scheduler events. We might use this to determine process lifetime and identify dependencies. You can learn more about what kind of "events" you can enable [here](https://lttng.org/man/1/lttng-enable-event/v2.8/). ```bash @@ -85,6 +87,12 @@ $ sudo lttng add-context --kernel --channel=channel0 --type=pid $ sudo lttng add-context --kernel --channel=channel0 --type=procname ``` +### Optionally, add CPU performance counters: +The Execution Events table shows how much each counter changed while a thread was running (Instruction Count, CPU Cycle, LLC Misses, Cache Misses). +```bash +$ sudo lttng add-context --kernel --channel=channel0 --type=perf:cpu:instructions --type=perf:cpu:cycles --type=perf:cpu:cache-misses --type=perf:cpu:LLC-load-misses +``` + ### Start the recording: ```bash $ sudo lttng start