Question about bit fields in bitpacked record lazarus / 62-General ---------------------------------------------------------------------- 440bx · 2021-06-05 06:27 Hello, I suspect that I am misunderstanding something about bitpacked records.  Referring to the attached image, the DWORD value is shown is $72.  I expected the field "unknown28" (the first 28 bits of that DWORD) to be $7, not $72 (which is the entire 32bit DWORD.)  Also, I expected the field "DepDisabled" to be $1 because the second bit in the DWORD is $1. The program that produces that output (which might be different on your system) is : Code: Pascal  [Select][+][-]window.onload = function(){var x1 = document.getElementById("main_content_section"); if (x1) { var x = document.getElementsByClassName("geshi");for (var i = 0; i {$APPTYPE      CONSOLE} {$TYPEDADDRESS ON} {$ifdef WIN64}  {$FATAL DEP policy cannot be queried in 64bit - compile for 32bit}{$endif} program _QueryInformationProcess; uses  Windows,  sysutils  ; {$ifdef VER90}type  DWORD = longint;  PDWORD = ^DWORD;{$endif} type  NTSTATUS = DWORD;   pboolean = ^boolean; const   { 22   34 } ProcessExecuteFlags = 34;  { TPROCESS_DEP_POLICY             }   ntdll = 'ntdll';   PROCESS_ALL_ACCESS        = STANDARD_RIGHTS_REQUIRED or SYNCHRONIZE or $FFFF;   STATUS_SUCCESS          = 0; type  { bit types for bitfields - since Delphi 2 does not support bitfields       }  {                           these types are FPC only                        }   T1BIT      = 0 ..        $1;  T2BITS     = 0 ..        $3;  T3BITS     = 0 ..        $7;  T4BITS   = 0 ..        $F;  T5BITS     = 0 ..       $1F;  T6BITS     = 0 ..     $3F;  T7BITS     = 0 ..       $7F;  T8BITS     = 0 ..       $FF;          { byte                              }  T9BITS     = 0 ..      $1FF;   T10BITS    = 0 ..      $3FF;  T11BITS    = 0 ..      $7FF;  T12BITS    = 0 ..      $FFF;  T13BITS    = 0 ..     $1FFF;  T14BITS    = 0 ..     $3FFF;  T15BITS    = 0 ..     $7FFF;  T16BITS    = 0 ..   $FFFF;            { word                              }  T17BITS    = 0 ..    $1FFFF;  T18BITS    = 0 ..    $3FFFF;  T19BITS    = 0 ..    $7FFFF;   T20BITS    = 0 ..    $FFFFF;  T21BITS    = 0 ..   $1FFFFF;  T22BITS    = 0 ..   $3FFFFF;  T23BITS    = 0 ..   $7FFFFF;  T24BITS    = 0 ..   $FFFFFF;  T25BITS    = 0 ..  $1FFFFFF;  T26BITS    = 0 ..  $3FFFFFF;  T27BITS    = 0 ..  $7FFFFFF;  T28BITS    = 0 ..  $FFFFFFF;  T29BITS    = 0 .. $1FFFFFFF;   T30BITS    = 0 .. $3FFFFFFF;  T31BITS    = 0 .. $7FFFFFFF; {$ifdef FPC}  type    { _PROCESS_DEP_POLICY                             - 32bit only          }     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;    TPROCESS_DEP_POLICY = bitpacked record      Unknown28                     : T28BITS;      Permanent                       : T1BIT;       DepEnabledThunk                 : T1BIT;      DepDisabled                   : T1BIT;      Unknown1                        : T1BIT;    end;{$endif} {$ifdef VER90}  type    { _PROCESS_DEP_POLICY                           - 32bit only            }     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;    TPROCESS_DEP_POLICY = record      ExecuteFlags          : DWORD;    end;{$endif} function NtQueryInformationProcess(ProcessHandle            : THANDLE;                                 ProcessInformationClass  : DWORD;                                 ProcessInformation       : pointer;                                 ProcessInformationLength : DWORD;                                 ReturnLength             : PDWORD)         : NTSTATUS; stdcall; external ntdll; function IsDebuggerPresent {$ifdef FPC} () {$endif}       : BOOL; stdcall external kernel32; function _itoa             (              { _in_  } In32bitInteger  : integer;              { _out_ } OutAsciizString : pchar;              { _in_  } InRadix         : DWORD             )         : pchar; cdecl external ntdll;                            { !! CDECL } function GetProcessDEPPolicy                   (                    { _in_  } InProcessHandle : THANDLE;                    { _out_ } OutFlags      : PDWORD;                    { _out_ } OutPermanent    : pboolean    { 4th bit set   }                   )       : BOOL; stdcall external kernel32; var  ProcessHandle                   : THANDLE; var  { variable for GetProcessDEPPolicy                                        }   Flags                           : DWORD;  Permanent                     : boolean; var  { variables for NtQueryInformationProcess                                 }   ProcessExecuteFlagsData         : TPROCESS_DEP_POLICY;   BufferSize    : DWORD;   ReturnLength  : DWORD;   Status        : NTSTATUS;    itoa_buf      : packed array[0..127] of char;  begin  writeln;  writeln;   // since we are dealing with our own process we can specify PROCESS_ALL_ACCESS   ProcessHandle := OpenProcess(PROCESS_ALL_ACCESS,                             FALSE,                               GetCurrentProcessId());   if ProcessHandle = 0 then Halt(1);   { use the API to verify the results                                         }   writeln;  writeln;  if not GetProcessDEPPolicy(ProcessHandle,                          @Flags,                            @Permanent) then  begin    { this shouldn't happen                                                 }    writeln;    writeln;    writeln('  GetProcessDEPPolicy failed.');  end;   writeln;  writeln('  Flags     = ', Flags);  writeln('  Permanent = ', Permanent);    { 4th bit of DWORD               }    { ---------------------------------------------------------- --------------- }  { get the same results using NtQueryInformationProcess                      }   writeln;  writeln;  writeln('  ProcessExecuteFlags : ', ord(ProcessExecuteFlags), ' (d) ',             IntToHex(ord(ProcessExecuteFlags), 0), ' (h)');   BufferSize   := sizeof(ProcessExecuteFlagsData);  ReturnLength := 0;   writeln;  writeln;  writeln('  Buffer size : ', BufferSize);   ZeroMemory(@ProcessExecuteFlagsData, sizeof(ProcessExecuteFlagsData));   if IsDebuggerPresent then DebugBreak();  Status := NtQueryInformationProcess(ProcessHandle,                                    ProcessExecuteFlags,                                   @ProcessExecuteFlagsData,                                    BufferSize,                                     @ReturnLength);   if IsDebuggerPresent then DebugBreak();  if Status <> STATUS_SUCCESS then  begin    writeln;    writeln;    writeln('  NtQueryInformationProcess failed.  NTSTATUS : ',               IntToHex(Status, 0));  end;   writeln;  writeln('  returned length : ', ReturnLength);    writeln;  writeln;   { output the result in binary for reference                                 }   writeln('  ', _itoa(DWORD(ProcessExecuteFlagsData), itoa_buf, 2));  writeln;    {$ifdef FPC}  with ProcessExecuteFlagsData do  begin    writeln('  ProcessExecuteFlagsData = $', IntToHex(DWORD(ProcessExecuteFlagsData), 0));    writeln;     writeln('  Unknown28       = $', IntToHex(Unknown28, 0));    writeln('  Permanent       = ',  Permanent);    writeln('  DepEnabledThunk = ',  DepEnabledThunk);    writeln('  DepDisabled     = ',  DepDisabled);  writeln('  Unknown1        = ',  Unknown1);  end;  {$endif}   {$ifdef VER90}    writeln;    writeln;    writeln('  Process execute flags : ',               IntToHex(ProcessExecuteFlagsData.ExecuteFlags, 0));  {$endif}     writeln;  writeln('Press / to end this program.');  readln;end.  My questions are: why isn't DepDisabled = 1 and why isn't Unknown28 = 7 ?  Any suggestion on how to get the result I want is definitely appreciated. Thank you for your help. ---------------------------------------------------------------------- MathMan · 2021-06-05 10:01 I may be off-track totally here, but shouldn't Code: Pascal  [Select][+][-]window.onload = function(){var x1 = document.getElementById("main_content_section"); if (x1) { var x = document.getElementsByClassName("geshi");for (var i = 0; i {$ifdef FPC}  type    { _PROCESS_DEP_POLICY                             - 32bit only            }     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;    TPROCESS_DEP_POLICY = bitpacked record      Unknown28                       : T28BITS;      Permanent                     : T1BIT;       DepEnabledThunk                 : T1BIT;      DepDisabled                     : T1BIT;      Unknown1                      : T1BIT;    end;{$endif} be defined as follows Code: Pascal  [Select][+][-]window.onload = function(){var x1 = document.getElementById("main_content_section"); if (x1) { var x = document.getElementsByClassName("geshi");for (var i = 0; i {$ifdef FPC}  type    { _PROCESS_DEP_POLICY                             - 32bit only            }     PPROCESS_DEP_POLICY = ^TPROCESS_DEP_POLICY;    TPROCESS_DEP_POLICY = bitpacked record      Unknown1                        : T1BIT;      DepDisabled                   : T1BIT;      DepEnabledThunk                 : T1BIT;      Permanent                       : T1BIT;      Unknown28                     : T28BITS;    end;{$endif} To me it looks like you have your definition upside down - that's why Unknown28 reflects the low 28 bit of the Flags value (when seen as DWORD). Cheers, MathMan ---------------------------------------------------------------------- Martin_fr · 2021-06-05 10:03 You are probably on a little endian machine (like intel)? So $00000072 writes to mem like this Record_addr: $72 $00 $00 $00 ---------------------------------------------------------------------- 440bx · 2021-06-05 10:38 @MathMan, @Martin_fr Yes, I am on a little endian machine (intel) I thought about what you both mentioned but, in that case, I would expect the "Unknown28" field to have the value $7200000 (28 bits) not just $72. IOW, the only values that make sense to me is either "0000000" (which is 0) or "7200000".  I don't see how "Unknown28" could be just "72".  In either endianness that value does not seem possible. ---------------------------------------------------------------------- MathMan · 2021-06-05 10:54 Quote from: 440bx on June 05, 2021, 12:38:31 pm@MathMan, @Martin_fr Yes, I am on a little endian machine (intel) I thought about what you both mentioned but, in that case, I would expect the "Unknown28" field to have the value $7200000 (28 bits) not just $72. IOW, the only values that make sense to me is either "0000000" (which is 0) or "7200000".  I don't see how "Unknown28" could be just "72".  In either endianness that value does not seem possible. Huh - why would you expect that? I would expect "Unknown28" to be little endian too on an Intel machine ... ---------------------------------------------------------------------- Martin_fr · 2021-06-05 11:05 Quote from: 440bx on June 05, 2021, 12:38:31 pm@MathMan, @Martin_fr Yes, I am on a little endian machine (intel) I thought about what you both mentioned but, in that case, I would expect the "Unknown28" field to have the value $7200000 (28 bits) not just $72. Not really, Lets name the bytes 11, 22, 33 and 44 Then lets look at the DWORD $44332211  and the packed subrange (fitting in 3 bytes) $332211 DWORD in mem: 11 22 33 44 That is so, if I access that mem as PByte I get $11 (the last byte of the DWORD $44332211) Or that I can do Code: Pascal  [Select][+][-]window.onload = function(){var x1 = document.getElementById("main_content_section"); if (x1) { var x = document.getElementsByClassName("geshi");for (var i = 0; i Fillbyte(mem, 0, MAXLEN);PByte(mem)^ := 7wirteln(PDWord(mem)^); // 7 Code: [Select]DWORD in mem: 11 22 33 44 RANGE in mem: 11 22 33 So if I read range from the DWORD address then i get the bytes 11 22 and 33, and in the variable they will have the value $332211 (the first byte from memory has the lowest significance and is the last byte of the value) And all that is intended. Your $00000072 Code: [Select]DWORD in mem: 72 00 00 00 RANGE in mem: 72 00 00 // 72 is the least significant byte, it will be at the end of the written number $000072 (that are 28 bits) The bytes in memory are always in reverse order to what they are in human readable representation. Off topic, you should love the German language. One of the few languages were spoken numbers contain digits in the wrong order. (Except for 13..19 which do that in many languages) English: 23 => Twenty-three German 23 => Drei-und-Zwanzig (three and twenty) But both have 14 => four-teen (the four move before the ten) ---------------------------------------------------------------------- 440bx · 2021-06-05 11:26 Quote from: MathMan on June 05, 2021, 12:54:43 pmHuh - why would you expect that? I would expect "Unknown28" to be little endian too on an Intel machine ... Because the compiler should be using one endianness or the other, not mix the two of them. if the compiler chooses big endian then, the value of "Unknown28" is $7200000 (28 bits), if it chooses little endian then the value of "Unknown28" is 0000000 which is 0 (zero).  There is no endianness that yields the value $72 for "Unknown28". I wrote a little test program to better see what is happening, its output is:Code: Text  [Select][+][-]window.onload = function(){var x1 = document.getElementById("main_content_section"); if (x1) { var x = document.getElementsByClassName("geshi");for (var i = 0; i i :  0  : 0i :  1  : 1i :  2  : 0i :  3  : 0i :  4  : 1i :  5  : 1i :  6  : 1i :  7  : 0i :  8  : 0i :  9  : 0i : 10  : 0i : 11  : 0i : 12  : 0i : 13  : 0i : 14  : 0i : 15  : 0i : 16  : 0i : 17  : 0i : 18  : 0i : 19  : 0i : 20  : 0i : 21  : 0i : 22  : 0i : 23  : 0i : 24  : 0i : 25  : 0i : 26  : 0i : 27  : 0i : 28  : 0i : 29  : 0i : 30  : 0i : 31  : 0  TwentyEight : 72Four        : 0[/size][/font]if the compiler is going to use the big endian memory ordering then the above shows that "Unknown28" should be $7200000 not $72 (that $72 is followed by 6 bytes of zeroes) and those cannot simply disappear because in big endian they are to the right of the value ($72 in this case). What the compiler seems to be doing is taking the first 28 bits, adding a nibble at the end and then evaluating the resulting 32bits in little endian mode, not big endian but, when doing individual bits it uses big endian.  IOW, it's mixing endianness-es depending on the field sizes. @Martin, I was typing this when you made your post.  I'm going to review it after posting this. the test program that produced that output is:Code: Pascal  [Select][+][-]window.onload = function(){var x1 = document.getElementById("main_content_section"); if (x1) { var x = document.getElementsByClassName("geshi");for (var i = 0; i {$APPTYPE    CONSOLE} {$TYPEDADDRESS ON}  program _BitFields2;  uses  Windows,  sysutils  ;  type  { bit types for bitfields - since Delphi 2 does not support bitfields       }  {                           these types are FPC only                        }   T1BIT      = 0 ..        $1;  T2BITS     = 0 ..        $3;  T3BITS     = 0 ..        $7;  T4BITS   = 0 ..        $F;  T5BITS     = 0 ..       $1F;  T6BITS     = 0 ..     $3F;  T7BITS     = 0 ..       $7F;  T8BITS     = 0 ..       $FF;          { byte                              }  T9BITS     = 0 ..      $1FF;   T10BITS    = 0 ..      $3FF;  T11BITS    = 0 ..      $7FF;  T12BITS    = 0 ..      $FFF;  T13BITS    = 0 ..     $1FFF;  T14BITS    = 0 ..     $3FFF;  T15BITS    = 0 ..     $7FFF;  T16BITS    = 0 ..   $FFFF;            { word                              }  T17BITS    = 0 ..    $1FFFF;  T18BITS    = 0 ..    $3FFFF;  T19BITS    = 0 ..    $7FFFF;   T20BITS    = 0 ..    $FFFFF;  T21BITS    = 0 ..   $1FFFFF;  T22BITS    = 0 ..   $3FFFFF;  T23BITS    = 0 ..   $7FFFFF;  T24BITS    = 0 ..   $FFFFFF;  T25BITS    = 0 ..  $1FFFFFF;  T26BITS    = 0 ..  $3FFFFFF;  T27BITS    = 0 ..  $7FFFFFF;  T28BITS    = 0 ..  $FFFFFFF;  T29BITS    = 0 .. $1FFFFFFF;   T30BITS    = 0 .. $3FFFFFFF;  T31BITS    = 0 .. $7FFFFFFF;  const  ntdll = 'ntdll';  function _itoa             (              { _in_  } In32bitInteger  : integer;              { _out_ } OutAsciizString : pchar;              { _in_  } InRadix         : DWORD             )       : pchar; cdecl external ntdll;                            { !! CDECL } var  BitTest                         : DWORD = $72;   BitFields                       : bitpacked record    Bits                            : bitpacked array[0..31] of T1BIT;  end absolute BitTest;   BitFields2                      : bitpacked record    case integer of      1 : (           TwentyEight                 : T28BITS;           Four                        : T4BITS;          );       2 : (           Bits                        : bitpacked array[0..31] of T1BIT;          );  end absolute BitTest;   i                             : integer;  begin  writeln;  writeln;   with BitFields2 do  begin    for i := low(Bits) to high(Bits) do    begin    writeln('  i : ', i:2, '  : ', Bits[i]);    end;     writeln;    writeln;    writeln('  TwentyEight : ', IntToHex(TwentyEight, 0));    writeln('  Four        : ', Four);  end;    writeln;  writeln('Press ENTER/RETURN to end this program.');  readln;end. ---------------------------------------------------------------------- 440bx · 2021-06-05 11:46 Quote from: Martin_fr on June 05, 2021, 01:05:03 pmAnd all that is intended. Your $00000072 Code: [Select]DWORD in mem: 72 00 00 00 RANGE in mem: 72 00 00 // 72 is the least significant byte, it will be at the end of the written number $000072 (that are 28 bits) I understand and agree with everything you've said but, as you showed, the "RANGE in mem" is 72 00 00 0 (28 bits), once the compiler decides to look at bits in memory order, it cannot change the way of looking at them.  In memory order the value is 72 00 00 0 Quote from: Martin_fr on June 05, 2021, 01:05:03 pmThe bytes in memory are always in reverse order to what they are in human readable representation. Completely agree with that.  what causes a real problem is that the compiler starts looking at bytes in memory-order and then switches to little-endian order which is what made the 72 00 00 0 show up as just 72. I'm going to have to review every bitpacked record definition I have and test thoroughly to ensure they behave as they should (the majority of them defined just as they are in C but, now I have doubts that they'll work as they do in C.) Thank you Martin and thank you MathMan for your help.  It was very useful. ---------------------------------------------------------------------- dseligo · 2021-06-05 11:47 Quote from: Martin_fr on June 05, 2021, 01:05:03 pmOff topic, you should love the German language. One of the few languages were spoken numbers contain digits in the wrong order. (Except for 13..19 which do that in many languages) English: 23 => Twenty-three German 23 => Drei-und-Zwanzig (three and twenty) Slovene also has this. But this is only for the last two digits. So there is also mixed endianness, like 440bx said ---------------------------------------------------------------------- howardpc · 2021-06-05 11:51 Quote from: Martin_fr on June 05, 2021, 01:05:03 pmOff topic, you should love the German language. One of the few languages were spoken numbers contain digits in the wrong order. (Except for 13..19 which do that in many languages) English: 23 => Twenty-three German 23 => Drei-und-Zwanzig (three and twenty) Poetic and Shakespearean English also has this, such as the nursery rhyme: "Four and twenty blackbirds baked in a pie"