namespace X86Disassembler.X86; using Operands; /// /// Handles decoding of ModR/M bytes in x86 instructions /// public class ModRMDecoder { // ModR/M byte masks private const byte MOD_MASK = 0xC0; // 11000000b private const byte REG_MASK = 0x38; // 00111000b private const byte RM_MASK = 0x07; // 00000111b // SIB byte masks private const byte SIB_SCALE_MASK = 0xC0; // 11000000b private const byte SIB_INDEX_MASK = 0x38; // 00111000b private const byte SIB_BASE_MASK = 0x07; // 00000111b // Register names for different sizes private static readonly string[] RegisterNames16 = {"ax", "cx", "dx", "bx", "sp", "bp", "si", "di"}; private static readonly string[] RegisterNames32 = {"eax", "ecx", "edx", "ebx", "esp", "ebp", "esi", "edi"}; // The instruction decoder that owns this ModRM decoder private readonly InstructionDecoder _decoder; /// /// Initializes a new instance of the ModRMDecoder class /// /// The instruction decoder that owns this ModRM decoder public ModRMDecoder(InstructionDecoder decoder) { _decoder = decoder; } /// /// Maps the register index from the ModR/M byte to the RegisterIndex enum value /// /// The register index from the ModR/M byte (0-7) /// The corresponding RegisterIndex enum value private RegisterIndex MapModRMToRegisterIndex(int modRMRegIndex) { // The mapping from ModR/M register index to RegisterIndex enum is: // 0 -> A (EAX) // 1 -> C (ECX) // 2 -> D (EDX) // 3 -> B (EBX) // 4 -> Sp (ESP) // 5 -> Bp (EBP) // 6 -> Si (ESI) // 7 -> Di (EDI) return modRMRegIndex switch { 0 => RegisterIndex.A, // EAX 1 => RegisterIndex.C, // ECX 2 => RegisterIndex.D, // EDX 3 => RegisterIndex.B, // EBX 4 => RegisterIndex.Sp, // ESP 5 => RegisterIndex.Bp, // EBP 6 => RegisterIndex.Si, // ESI 7 => RegisterIndex.Di, // EDI _ => RegisterIndex.A // Default to EAX }; } /// /// Maps the register index from the ModR/M byte to the RegisterIndex8 enum value /// /// The register index from the ModR/M byte (0-7) /// The corresponding RegisterIndex8 enum value private RegisterIndex8 MapModRMToRegisterIndex8(int modRMRegIndex) { // The mapping from ModR/M register index to RegisterIndex8 enum is direct: // 0 -> AL, 1 -> CL, 2 -> DL, 3 -> BL, 4 -> AH, 5 -> CH, 6 -> DH, 7 -> BH return (RegisterIndex8)modRMRegIndex; } /// /// Decodes a ModR/M byte to get the operand /// /// The mod field (2 bits) /// The r/m field as RegisterIndex /// True if the operand is 64-bit /// The operand object public Operand DecodeModRM(byte mod, RegisterIndex rmIndex, bool is64Bit) { int operandSize = is64Bit ? 64 : 32; switch (mod) { case 0: // [reg] or disp32 // Special case: [EBP] is encoded as disp32 with no base register if (rmIndex == RegisterIndex.Bp) // disp32 (was EBP/BP) { if (_decoder.CanReadUInt()) { uint disp32 = _decoder.ReadUInt32(); return OperandFactory.CreateDirectMemoryOperand(disp32, operandSize); } // Fallback for incomplete data return OperandFactory.CreateDirectMemoryOperand(0, operandSize); } // Special case: [ESP] is encoded with SIB byte if (rmIndex == RegisterIndex.Sp) // SIB (was ESP/SP) { // Handle SIB byte if (_decoder.CanReadByte()) { byte sib = _decoder.ReadByte(); return DecodeSIB(sib, 0, is64Bit); } // Fallback for incomplete data return OperandFactory.CreateBaseRegisterMemoryOperand(RegisterIndex.Sp, operandSize); } // Regular case: [reg] return OperandFactory.CreateBaseRegisterMemoryOperand(rmIndex, operandSize); case 1: // [reg + disp8] if (rmIndex == RegisterIndex.Sp) // SIB + disp8 (ESP/SP) { // Handle SIB byte if (_decoder.CanReadByte()) { byte sib = _decoder.ReadByte(); sbyte disp8 = (sbyte)(_decoder.CanReadByte() ? _decoder.ReadByte() : 0); return DecodeSIB(sib, (uint)disp8, is64Bit); } // Fallback for incomplete data return OperandFactory.CreateBaseRegisterMemoryOperand(RegisterIndex.Sp, operandSize); } else { if (_decoder.CanReadByte()) { sbyte disp8 = (sbyte)_decoder.ReadByte(); // For EBP (BP), always create a displacement memory operand, even if displacement is 0 // This is because [EBP] with no displacement is encoded as [EBP+0] if (disp8 == 0 && rmIndex != RegisterIndex.Bp) { return OperandFactory.CreateBaseRegisterMemoryOperand(rmIndex, operandSize); } return OperandFactory.CreateDisplacementMemoryOperand(rmIndex, disp8, operandSize); } // Fallback for incomplete data return OperandFactory.CreateBaseRegisterMemoryOperand(rmIndex, operandSize); } case 2: // [reg + disp32] if (rmIndex == RegisterIndex.Sp) // SIB + disp32 (ESP/SP) { // Handle SIB byte if (_decoder.CanReadUInt()) { byte sib = _decoder.ReadByte(); uint disp32 = _decoder.ReadUInt32(); return DecodeSIB(sib, disp32, is64Bit); } // Fallback for incomplete data return OperandFactory.CreateBaseRegisterMemoryOperand(RegisterIndex.Sp, operandSize); } else { if (_decoder.CanReadUInt()) { uint disp32 = _decoder.ReadUInt32(); // For EBP (BP), always create a displacement memory operand, even if displacement is 0 // This is because [EBP] with no displacement is encoded as [EBP+disp] if (rmIndex == RegisterIndex.Bp) { return OperandFactory.CreateDisplacementMemoryOperand(rmIndex, (int)disp32, operandSize); } // Only show displacement if it's not zero if (disp32 == 0) { return OperandFactory.CreateBaseRegisterMemoryOperand(rmIndex, operandSize); } return OperandFactory.CreateDisplacementMemoryOperand(rmIndex, (int)disp32, operandSize); } // Fallback for incomplete data return OperandFactory.CreateBaseRegisterMemoryOperand(rmIndex, operandSize); } case 3: // reg (direct register access) return OperandFactory.CreateRegisterOperand(rmIndex, operandSize); default: // Fallback for invalid mod value return OperandFactory.CreateRegisterOperand(RegisterIndex.A, operandSize); } } /// /// Peaks a ModR/M byte and returns the raw field values, without advancing position /// /// A tuple containing the raw mod, reg, and rm fields from the ModR/M byte public byte PeakModRMReg() { if (!_decoder.CanReadByte()) { return 0; } byte modRM = _decoder.PeakByte(); // Extract fields from ModR/M byte byte regIndex = (byte)((modRM & REG_MASK) >> 3); // Middle 3 bits (bits 3-5) return regIndex; } /// /// Reads a ModR/M byte and returns the raw field values /// /// A tuple containing the raw mod, reg, and rm fields from the ModR/M byte public (byte mod, byte reg, byte rm) ReadModRMRaw() { if (!_decoder.CanReadByte()) { return (0, 0, 0); } byte modRM = _decoder.ReadByte(); // Extract fields from ModR/M byte byte mod = (byte)((modRM & MOD_MASK) >> 6); // Top 2 bits (bits 6-7) byte regIndex = (byte)((modRM & REG_MASK) >> 3); // Middle 3 bits (bits 3-5) byte rmIndex = (byte)(modRM & RM_MASK); // Bottom 3 bits (bits 0-2) return (mod, regIndex, rmIndex); } /// /// Reads and decodes a ModR/M byte for standard 32-bit operands /// /// A tuple containing the mod, reg, rm fields and the decoded operand public (byte mod, RegisterIndex reg, RegisterIndex rm, Operand operand) ReadModRM() { return ReadModRMInternal(false); } /// /// Reads and decodes a ModR/M byte for 64-bit operands /// /// A tuple containing the mod, reg, rm fields and the decoded operand public (byte mod, RegisterIndex reg, RegisterIndex rm, Operand operand) ReadModRM64() { return ReadModRMInternal(true); } /// /// Reads and decodes a ModR/M byte for 8-bit operands /// /// A tuple containing the mod, reg, rm fields and the decoded operand public (byte mod, RegisterIndex8 reg, RegisterIndex8 rm, Operand operand) ReadModRM8() { return ReadModRM8Internal(); } /// /// Internal implementation for reading and decoding a ModR/M byte for standard 32-bit or 64-bit operands /// /// True if the operand is 64-bit /// A tuple containing the mod, reg, rm fields and the decoded operand private (byte mod, RegisterIndex reg, RegisterIndex rm, Operand operand) ReadModRMInternal(bool is64Bit) { if (!_decoder.CanReadByte()) { return (0, RegisterIndex.A, RegisterIndex.A, OperandFactory.CreateRegisterOperand(RegisterIndex.A, is64Bit ? 64 : 32)); } byte modRM = _decoder.ReadByte(); // Extract fields from ModR/M byte byte mod = (byte)((modRM & MOD_MASK) >> 6); byte regIndex = (byte)((modRM & REG_MASK) >> 3); byte rmIndex = (byte)(modRM & RM_MASK); // Map the ModR/M register indices to RegisterIndex enum values RegisterIndex reg = MapModRMToRegisterIndex(regIndex); RegisterIndex rm = MapModRMToRegisterIndex(rmIndex); // Create the operand based on the mod and rm fields Operand operand = DecodeModRM(mod, rm, is64Bit); return (mod, reg, rm, operand); } /// /// Internal implementation for reading and decoding a ModR/M byte for 8-bit operands /// /// A tuple containing the mod, reg, rm fields and the decoded operand private (byte mod, RegisterIndex8 reg, RegisterIndex8 rm, Operand operand) ReadModRM8Internal() { if (!_decoder.CanReadByte()) { return (0, RegisterIndex8.AL, RegisterIndex8.AL, OperandFactory.CreateRegisterOperand8(RegisterIndex8.AL)); } byte modRM = _decoder.ReadByte(); // Extract fields from ModR/M byte byte mod = (byte)((modRM & MOD_MASK) >> 6); byte regIndex = (byte)((modRM & REG_MASK) >> 3); byte rmIndex = (byte)(modRM & RM_MASK); // Map the ModR/M register indices to RegisterIndex8 enum values RegisterIndex8 reg = MapModRMToRegisterIndex8(regIndex); RegisterIndex8 rm = MapModRMToRegisterIndex8(rmIndex); // Create the operand based on the mod and rm fields Operand operand; if (mod == 3) // Register operand { // For register operands, create an 8-bit register operand operand = OperandFactory.CreateRegisterOperand8(rm); } else // Memory operand { // For memory operands, we need to map the RegisterIndex8 to RegisterIndex for base registers RegisterIndex rmRegIndex = MapRegister8ToBaseRegister(rm); operand = DecodeModRM(mod, rmRegIndex, false); operand.Size = 8; // Set size to 8 bits } return (mod, reg, rm, operand); } /// /// Decodes a SIB byte /// /// The SIB byte /// The displacement value /// True if the operand is 64-bit /// The decoded SIB operand private Operand DecodeSIB(byte sib, uint displacement, bool is64Bit) { int operandSize = is64Bit ? 64 : 32; // Extract fields from SIB byte byte scale = (byte)((sib & SIB_SCALE_MASK) >> 6); int indexIndex = (sib & SIB_INDEX_MASK) >> 3; int baseIndex = sib & SIB_BASE_MASK; // Map the SIB register indices to RegisterIndex enum values RegisterIndex index = MapModRMToRegisterIndex(indexIndex); RegisterIndex @base = MapModRMToRegisterIndex(baseIndex); // Special case: ESP/SP (4) in index field means no index register if (index == RegisterIndex.Sp) { // Special case: EBP/BP (5) in base field with no displacement means disp32 only if (@base == RegisterIndex.Bp && displacement == 0) { if (_decoder.CanReadUInt()) { uint disp32 = _decoder.ReadUInt32(); // When both index is ESP (no index) and base is EBP with disp32, // this is a direct memory reference [disp32] return OperandFactory.CreateDirectMemoryOperand(disp32, operandSize); } // Fallback for incomplete data return OperandFactory.CreateDirectMemoryOperand(0, operandSize); } // When index is ESP (no index), we just have a base register with optional displacement if (displacement == 0) { return OperandFactory.CreateBaseRegisterMemoryOperand(@base, operandSize); } return OperandFactory.CreateDisplacementMemoryOperand(@base, (int)displacement, operandSize); } // Special case: EBP/BP (5) in base field with no displacement means disp32 only if (@base == RegisterIndex.Bp && displacement == 0) { if (_decoder.CanReadUInt()) { uint disp32 = _decoder.ReadUInt32(); int scaleValue = 1 << scale; // 1, 2, 4, or 8 // If we have a direct memory reference with a specific displacement, // use a direct memory operand instead of a scaled index memory operand if (disp32 > 0 && index == RegisterIndex.Sp) { return OperandFactory.CreateDirectMemoryOperand(disp32, operandSize); } // Create a scaled index memory operand with displacement but no base register return OperandFactory.CreateScaledIndexMemoryOperand( index, scaleValue, null, (int)disp32, operandSize); } // Fallback for incomplete data return OperandFactory.CreateScaledIndexMemoryOperand( index, 1 << scale, null, 0, operandSize); } // Normal case with base and index registers int scaleFactor = 1 << scale; // 1, 2, 4, or 8 // Create a scaled index memory operand return OperandFactory.CreateScaledIndexMemoryOperand( index, scaleFactor, @base, (int)displacement, operandSize); } /// /// Gets the register name based on the register index and size /// /// The register index as RegisterIndex enum /// The register size (16 or 32 bits) /// The register name public static string GetRegisterName(RegisterIndex regIndex, int size) { return size switch { 16 => RegisterNames16[(int)regIndex], 32 => RegisterNames32[(int)regIndex], 64 => RegisterNames32[(int)regIndex], // For now, reuse 32-bit names for 64-bit _ => "unknown" }; } /// /// Gets the 8-bit register name based on the RegisterIndex8 enum value /// /// The register index as RegisterIndex8 enum /// The 8-bit register name public static string GetRegisterName(RegisterIndex8 regIndex8) { return regIndex8.ToString().ToLower(); } /// /// Maps a RegisterIndex8 enum value to the corresponding RegisterIndex enum value for base registers /// /// The RegisterIndex8 enum value /// The corresponding RegisterIndex enum value private RegisterIndex MapRegister8ToBaseRegister(RegisterIndex8 regIndex8) { // Map 8-bit register indices to their corresponding 32-bit register indices return regIndex8 switch { RegisterIndex8.AL => RegisterIndex.A, RegisterIndex8.CL => RegisterIndex.C, RegisterIndex8.DL => RegisterIndex.D, RegisterIndex8.BL => RegisterIndex.B, RegisterIndex8.AH => RegisterIndex.A, RegisterIndex8.CH => RegisterIndex.C, RegisterIndex8.DH => RegisterIndex.D, RegisterIndex8.BH => RegisterIndex.B, _ => RegisterIndex.A // Default to EAX }; } }