Direct use of OpenSSL without using intermediate libraries (LNet,
Indy, Synapse)
lazarus / 38-Networking and Web Programming

----------------------------------------------------------------------
delphius · 2023-09-15 13:13

Please, pros, explain to me why, when developing Free Pascal (Lazarus)
programs, we must necessarily rely on intermediary libraries like
LNet, Indy, Synapse, etc.?
Why is it impossible to work with OpenSSL directly for any tasks
requiring tls/ssl, as tfphttpclient does, just using fpc openssl unit?

How will such code from the example working with OpenSSL over simple
sockets directly be worse?
Code: Pascal  [Select][+][-]window.onload = function(){var x1 =
document.getElementById("main_content_section"); if (x1) { var x =
document.getElementsByClassName("geshi");for (var i = 0; i program
opensslget; {$mode objfpc}{$H+}{$Packrecords C} uses  SysUtils,
sockets, resolve, openssl; type  PSSL_CIPHER = ^SSL_CIPHER;  // https:
//github.com/openssl/openssl/blob/ed4a71d18d939f557b430c4369d118b55c1c
0b6c/ssl/ssl_local.h#L397  SSL_CIPHER = record    valid: LongWord;   
name: PChar;    stdname: PChar;    id: packed array[0..1] of Char;   
algorithm_mkey: LongWord;    algorithm_auth: LongWord;   
algorithm_enc: LongWord;    algorithm_mac: LongWord;    min_tls:
Integer;    max_tls: Integer;    min_dtls: Integer;    max_dtls:
Integer;    algo_strength: LongWord;    algorithm2: LongWord;   
strength_bits: LongInt;    alg_bits: LongWord;  end; const  // The
read functions work based on the SSL/TLS records. The data are
received in records (with a maximum record size of 16kB)  // For TLS
1.2 and earlier, that limit is 2^14 octets. TLS 1.3 uses a limit of
2^14+1 octets.  // https://www.rfc-
editor.org/rfc/rfc8449.html#section-4  BUF_SIZE = 16 * 1024;   Host =
'example.org';  Port = 443;   {$IFDEF WINDOWS}  ext = '.dll'; 
{$ELSE}  ext = '.so.3';  {$ENDIF} var  filepath, trustedCertFile:
string;  //OpenSSL  ctx: PSSL_CTX;  ssl: PSSL;  cipherPtr:
PSSL_CIPHER;  server_cert:pX509;  in_buf: array[0..BUF_SIZE - 1] of
char;  request: string;  nbytes_written, nbytes_read, ssl_error:
integer;  cert_str: AnsiString;  tempBuf: AnsiString;  //Sockets 
CSocket: TSocket;  Address: TInetSockAddr;  IPAddr: string;  hrs:
THostResolver; procedure PrintSSLCipherInfo(const cipher:
SSL_CIPHER);begin  Writeln('Cipher Name: ', cipher.name); 
Writeln('RFC Name: ', cipher.stdname);  //
https://www.iana.org/assignments/tls-parameters/tls-
parameters.xhtml#tls-parameters-4  Writeln('TLS Cipher ID: ', '0x' +
IntToHex(Ord(cipher.id[1]), 2) + ', 0x' + IntToHex(Ord(cipher.id[0]),
2));  Writeln('Key Exchange Algorithm: ', cipher.algorithm_mkey); 
Writeln('Server Authentication Algorithm: ', cipher.algorithm_auth); 
Writeln('Symmetric Encryption Algorithm: ', cipher.algorithm_enc); 
Writeln('Symmetric Authentication Algorithm: ',
cipher.algorithm_mac);  Writeln('Minimum SSL/TLS Protocol Version: ',
cipher.min_tls);  Writeln('Maximum SSL/TLS Protocol Version: ',
cipher.max_tls);  Writeln('Minimum DTLS Protocol Version: ',
cipher.min_dtls);  Writeln('Maximum DTLS Protocol Version: ',
cipher.max_dtls);  Writeln('Strength and Export Flags: ',
cipher.algo_strength);  Writeln('Extra Flags: ', cipher.algorithm2); 
Writeln('Number of Bits Used: ', cipher.strength_bits); 
Writeln('Number of Bits for Algorithm: ', cipher.alg_bits);end; begin 
filepath := ExtractFilePath(ParamStr(0));  //
https://curl.se/ca/cacert.pem  trustedCertFile := filepath +
'cacert.pem';  // Load OpenSSL dynamic libraries from program folder 
InitSSLInterface(filepath + 'libssl' + ext, filepath + 'libcrypto' +
ext);   // Show libs version https://github.com/openssl/openssl/blob/8
f51b2279eda1e0cffb3400c2e5b5c3771f62ea7/include/openssl/crypto.h.in#L1
62  WriteLn('OpenSSL version: ' + OpenSSLGetVersion(7)); //
OPENSSL_FULL_VERSION_STRING    // Create an SSL context.  ctx :=
SSLCTXnew(SslTLSMethod);  if not Assigned(ctx) then  begin   
WriteLn('Could not create SSL context');    Halt(1);  end;   // Load
trusted certificates.  if SSLCTXloadverifylocations(ctx,
PChar(trustedCertFile), '') <= 0 then    begin      WriteLn('Could not
load trusted certificates');      Halt(1);    end;   // Set
verification options.  SSLCTXsetverify(ctx, SSL_VERIFY_PEER, nil); 
SSLCTXsetmode(ctx, SSL_MODE_AUTO_RETRY);   // Create SSL connection 
ssl := SSLnew(ctx);  if ssl = nil then  begin    Writeln('Error
creating SSL connection');    Halt(1);  end;   // Create simple
socket  CSocket := fpsocket(AF_INET, SOCK_STREAM, 0);  if CSocket = -1
then  begin    WriteLn('Error creating socket');    Halt(1);  end;  
// Resolve domain name to IP adress  hrs :=
THostResolver.Create(nil);  if hrs.NameLookup(Host) then  begin   
IPAddr := hrs.AddressAsString;    WriteLn('Resolved IP Address: ',
IPAddr);  end  else  begin    WriteLn('Failed to resolve the hostname:
', Host);    Halt(1);  end;   // Connect to host  with Address do 
 begin      sin_family := AF_INET; //TCP/IP      sin_port:=
htons(word(Port)); //Port     
sin_addr:=StrToNetAddr(hrs.AddressAsString); // IP address   end;  
hrs.Free;   if fpconnect(CSocket, @Address, SizeOf(Address)) < 0 then 
begin    WriteLn('Error connecting to server.');    Halt(1);  end;  
writeln('Connected ', Host, ':', Port);   // Set file descriptor (fd)
of connected socket to SSL connection  SSLsetfd(ssl, CSocket);    //
Establish SSL connection  if SslConnect(ssl) <= 0 then  begin   
Writeln('SSL connection error.');    Halt(1);  end;  Writeln('SSL
connection established.');  WriteLn('----' + #13#10 + 'SSL connection
cipher description:');  //
https://www.openssl.org/docs/manmaster/man3/SSL_get_cipher.html 
cipherPtr := SSLgetcurrentcipher(ssl); 
PrintSSLCipherInfo(cipherPtr^);  WriteLn('----');  //
https://www.openssl.org/docs/manmaster/man3/SSL_get_version.html 
WriteLn('SSL/TLS version: ' + SslGetVersion(ssl));  WriteLn('----'); 
// https://www.openssl.org/docs/manmaster/man3/SSL_get_peer_certificat
e.html  server_cert := SslGetPeerCertificate(ssl);  WriteLn('Server
certificate:');  SetLength(tempBuf, 1024);  //
https://www.openssl.org/docs/man3.0/man3/X509_NAME_oneline.html  // ht
tps://www.openssl.org/docs/manmaster/man3/X509_get_subject_name.html 
cert_str := X509NameOneline(X509GetSubjectName(server_cert), tempBuf,
Length(tempBuf));  cert_str := StringReplace(cert_str, '\xC2\xA0', '
', [rfReplaceAll]);  WriteLn('Subject: ', cert_str);  //
https://www.openssl.org/docs/manmaster/man3/X509_get_issuer_name.html 
cert_str := X509NameOneline(X509GetIssuerName(server_cert), tempBuf,
Length(tempBuf));  cert_str := StringReplace(cert_str, '\xC2\xA0', '
', [rfReplaceAll]);  WriteLn('Issuer: ', cert_str + #13#10);   // We
could do all sorts of certificate verification stuff here before
deallocating the certificate  //
https://www.openssl.org/docs/manmaster/man3/X509_free.html 
X509free(server_cert);   // Create an HTTP GET request.  request :=  
'GET / HTTP/1.1'#13#10 +    'Host: ' + Host + #13#10 +    'Connection:
close'#13#10 +    'User-Agent: Example TLS client'#13#10#13#10;   //
Send the request to the server.  WriteLn('--> Sending to the
server:');  Write(request);  //
https://www.openssl.org/docs/manmaster/man3/SSL_write.html 
nbytes_written := SslWrite(ssl, PChar(request), Length(request));  if
nbytes_written <> Length(request) then  begin    WriteLn('Could not
send all data to the server');    Halt(1);  end;   WriteLn('-->
Sending to the server finished');  WriteLn('<-- Receiving from the
server:');   // Receive and display the server's response.  repeat   
// https://www.openssl.org/docs/manmaster/man3/SSL_read.html   
nbytes_read := SslRead(ssl, @in_buf, BUF_SIZE);    if nbytes_read <= 0
then    begin      ssl_error := SSLgeterror(ssl, nbytes_read);      if
ssl_error = SSL_ERROR_ZERO_RETURN then        Break      else     
begin        WriteLn('Error ', ssl_error, ' while reading data from
the server');        Halt(1);      end;    end;    
Write(UTF8Encode(Copy(in_buf, 0, nbytes_read)));  until False;  
WriteLn('<-- Receiving from the server finished');    // Send SSL/TLS
close_notify  //
https://www.openssl.org/docs/manmaster/man3/SSL_shutdown.html  repeat 
  ssl_error := SslShutdown(ssl);    until ssl_error <> 0;   if
ssl_error = 1 then      WriteLn('SSL connection sussefully closed.') 
 else      //
https://www.openssl.org/docs/manmaster/man3/SSL_get_error.html     
WriteLn('SSL connection closed with error.',
Err_Error_String(SSLgeterror(ssl, ssl_error), Nil));   // Close
socket  CloseSocket(CSocket);   // Clean up.  if Assigned(ssl) then  
SslFree(ssl);   if Assigned(ctx) then    SSLCtxFree(ctx);end.

----------------------------------------------------------------------
rvk · 2023-09-15 13:25

Quote from: delphius on September 15, 2023, 03:13:16 pmHow will such
code from the example working with OpenSSL over simple sockets
directly be worse?
Won't work on Windows 64 bit with FPC/Lazarus 64 bit.
Won't work on Linux (any bitness).
Won't work on... etc.

Using intermediate libraries takes so much work out of your hands.
I won't even begin to describe how you would need to explain this to
beginners.
If I want to show you how to do something, I can show you code, using
one of those libraries, without worrying what platform you are on.
The code you showed only works on one platform (so not cross-
platform).

It works, but is only for a small portion of users.

Quote from: delphius on September 15, 2023, 03:13:16 pmWhy is it
impossible to work with OpenSSL directly for any tasks requiring
tls/ssl, as tfphttpclient does, just using fpc openssl unit?
You said that earlier. But I'm not sure if I understand your
objections.
fphttpclient in combination with openssl is just one way.
BTW. The openssl from FPC comes from Synapse !!

Synpase with openssl is just another way.

LNet is another way.

Indy is another way.

All have their own upsides and downsides. But they are all examples of
communications library in combination with some sort of ssl library.
And most are cross-platform and ready to use.

----------------------------------------------------------------------
delphius · 2023-09-15 13:48

Quote from: rvk on September 15, 2023, 03:25:18 pmWon't work on
Windows 64 bit with FPC/Lazarus 64 bit.
Won't work on Linux (any bitness).
Won't work on... etc.

Well, that's how work on all intermediary libraries began, isn't it? 

Quote from: rvk on September 15, 2023, 03:25:18 pmThe code you showed
only works on one platform (so not cross-platform).

Well, technically it compiles and works on both Windows and Linux. And
this is just an example, not the basis of a real library.

And I am interested in the opinion of professionals and experienced
users, to whom I do not consider myself in any way.

Thank you for your opinion! 

Quote from: rvk on September 15, 2023, 03:25:18 pmBTW. The openssl
from FPC comes from Synapse !!

Yes, I understand this perfectly well, but I relied on it precisely
because it already exists and it is built into fpc, although it
requires global reworking

Quote from: rvk on September 15, 2023, 03:25:18 pmBut they are all
examples of communications library in combination with some sort of
ssl library. And most are cross-platform and ready to use.

I would like to have such a library in the standard delivery of the
fpc that works reliably and cross-platform out of the box without
dancing with a tambourine.
Maybe I want a lot?

----------------------------------------------------------------------
rvk · 2023-09-15 14:51

Quote from: delphius on September 15, 2023, 03:48:35 pmI would like to
have such a library in the standard delivery of the fpc that works
reliably and cross-platform out of the box without dancing with a
tambourine.
Maybe I want a lot? 
True... but remember... when including such library in FPC in the FCL,
it can't be easily recompiled again.
So new functionality can't be easily added (as you have seen).

openssl.pas from FPC is perfectly fine (except that DLLVersions is
hard coded and needs to be 'hacked' to privide newer functionality).

But if you are going to include a new_openssl.pas in FPC in the FCL,
it is compiled in automatically and can't be changed anymore (easily).

The only way out of this is include it as extra package (which you can
recompile), NOT part of the FCL from FPC itself.

And that's exactly how Synapse, LNet and Indy do it. So using the
latest version of those libraries you have always the latest
functionality.

But there is nothing wrong with developing your own delcom package
(delphius communication package) which can be included in the OPM for
easy addition to FPC/Lazarus

So you would need to make the decision:
Inside the FCL and delivered with FPC but static and you can't change
it easily
or included as added package via OPM and you can always provide the
latest version more easily.

----------------------------------------------------------------------
delphius · 2023-09-15 19:37

Quote from: rvk on September 15, 2023, 04:51:47 pmBut there is nothing
wrong with developing your own delcom package (delphius communication
package) which can be included in the OPM for easy addition to
FPC/Lazarus

  thank you so much for the offer, but I would not like to invent
another bike, those that already exist ride great

I see my tactics in small pinpoint strikes on gaps in functionality
based on my own needs in the first place, trying not to use external
dependencies at all, except for fpc and not to generate new ones,
creating a single base that will pull a loop of dependencies with
increasing levels of abstraction and functionality.

Quote from: rvk on September 15, 2023, 04:51:47 pmInside the FCL and
delivered with FPC but static and you can't change it easily
or included as added package via OPM and you can always provide the
latest version more easily.

Of course, the second option looks more acceptable, but who prevents
combining both options, if in fact there will be a binding only to
standard fpc modules and there will be no own core, then you can
simply include in the fcl those parts that have reached maximum
stability and maturity, leaving the parts that need improvement or
dynamic parts in an external package

Quote from: rvk on September 15, 2023, 04:51:47 pmpackage via OPM

And then the second painful question pops up, which torments me and to
which I do not find an answer 

I (oh horror  ) do not use Lazarus for development
I spend 90 percent of my coding time in VSCode + omnipascal + fpc +
gdb. Ooops.
Where is the cli version of opm wich I can use? (I know about python
CLI package manager for Lazarus from respectable Warfley)
Where is the package manager for fpc like nuget/npm/pip/ect or vcpkg
at worst? 
fppkg? Ok, where its online repo with all this beautiful things I've
found every day on github(lab)? (I know about FPPKG Repository Server
from respectable Joost van der Sluis)

These are all questions without answers to myself

----------------------------------------------------------------------
Thaddy · 2023-09-16 12:33

Well, our member Xor-el wrote a compatible replacement in ALL pure
Pascal - so not an interface - some years ago. Yoy do not need to rely
on Openssl etc, because once you adhere to the standards it can - and
has been - written in pure Pascal.
It is merely the usual laziness that prohibits some people to use the
source.... (   )

(But then again, Xor-el's real name is rather unpronounceable for
anybody but Nigerians   )
Google his code. It is on gitbub. And he is a computer scientist.

----------------------------------------------------------------------
maurobio · 2023-09-16 13:47

Hi, Thaddy!

Here is Xor-el's GitHub page:

https://github.com/Xor-el

Could you please tell us precisely what is the library you cited?
There are several there.

Thank you!

With best regards,

----------------------------------------------------------------------
delphius · 2023-09-16 14:09

Quote from: maurobio on September 16, 2023, 03:47:35 pmHi, Thaddy!

Here is Xor-el's GitHub page:

https://github.com/Xor-el

Could you please tell us precisely what is the library you cited?
There are several there.

Thank you!

With best regards,

He means https://github.com/Xor-el/CryptoLib4Pascal , Its in my
bookmarks for the future since a long time ago.
But the problem is, that we need to realise whole TLSv1.3 protocol
base on it, and this is solvable, but quite difficult task 
It can be compared with a new net lib writing with all its
difficulties and underwater stones

----------------------------------------------------------------------
maurobio · 2023-09-16 14:20

Ave, @delphius!

QuoteBut the problem is, that we need to realise whole TLSv1.3
protocol base on it, and this is solvable, but quite difficult task
I see. Along with the problems already pointed out by @rvk, this whole
situation looks more and more like a "Catch-22" condition (as
wonderfully described in Joseph Heller's novel of the same name).

With warmest regards,

----------------------------------------------------------------------
toby · 2023-09-17 20:06

delphius

i'd love to get your  opensslget code working that you posted in the
topic

i have fpc-3.3.1 installed and i have OpenSSL 3.0.5 installed but
still don't get anything from the opensslgetversion line

what can i look for?

----------------------------------------------------------------------
delphius · 2023-09-17 20:37

Quote from: toby on September 17, 2023, 10:06:40 pm
delphius

i'd love to get your  opensslget code working that you posted in the
topic

i have fpc-3.3.1 installed and i have OpenSSL 3.0.5 installed but
still don't get anything from the opensslgetversion line

what can i look for?

Let's first check that this code will work for you if libssl.so.3 and
libcrypto.so.3 of the latest version (3.1.2) are in the program
folder.

Build them from the source code or I can send you the libraries I have
already compiled for linux.

For the program to pick them up, or force load them using
InitSSLInterface(filepath + 'libssl' + ext, filepath + 'libcrypto' +
ext); or use the alternative method I suggested

If you succeed and the code works, then we will deal further, already
with your specific linux distribution and libraries.

Remember that I am constantly modifying the code in the first message,
so for the purity of the experiment, copy it again from the first
message of the topic

----------------------------------------------------------------------
toby · 2023-09-17 21:12

delphius

it works great i had to assign filepath varoable to the REAL location
of my libs

/usr/local/lib64/libssl.so.3
/usr/local/lib64/libcrypto.so.3

i had thought they were in /usr/local/lib/ and not where they really
were in /usr/local/lib64
thanks for making me check - i never thought to verify their location
for almost 2 days
i thought the problem was in openssl.pas so you should see how i
butchered it up

very nice program also    works on my own server perfectly and maybe
faster then curl

---

just a question if you know
what on earth is opensslsockets unit used for?

----------------------------------------------------------------------
delphius · 2023-09-18 05:54

Quote from: toby on September 17, 2023, 11:12:02 pmi thought the
problem was in openssl.pas so you should see how i butchered it up
I'm glad you did it! 

Quote from: toby on September 17, 2023, 11:12:02 pmvery nice program
also    works on my own server perfectly and maybe faster then curl
It's just a POC (proof of concept) its insecure & buggy  I do not
recommend to use it in production, just for testing
fphttpclient makes this work perfectly

Quote from: toby on September 17, 2023, 11:12:02 pmjust a question if
you know
what on earth is opensslsockets unit used for?

Its just a higher level pascal wraper to OpenSSL lib, to improve the
easiness of using it.
You can use openssl.pas directly, or thru the higer level of
abstraction with opensslsockets.pas
Or even you just can transpile last OpenSSL c header file, and work
max close to OpenSSL lib

Its just like difference between socket and ssocket units

----------------------------------------------------------------------
toby · 2023-09-18 18:01

a suggestion why not do your new code as attachment in a new post
reply - so someone just coming upon this topic can see your program
developement etc

i saved the original opensslget.pas (that became opensslmail.pas and
now this new opensslget.pas

easier to see a new post and not waste bandwidth downloading the first
post to see if
changes were made

i check the time of the modification but  they use 'today' and gmt
time?  so i really have no idea whose 'today' it is

---

i made a separate pempath as i doubt anyone will keep their .pem in
the same dir as the libssl/libcrypto libs

---

Hint: Variable "tempBuf" of a managed type does not seem to be
initialized

i added tempBuf := ''; above it's setlength
tempBuf := '';
      SetLength(tempBuf, 1024);

----------------------------------------------------------------------
delphius · 2023-09-27 13:17

Quote from: toby on September 18, 2023, 08:01:10 pma suggestion why
not do your new code as attachment in a new post reply - so someone
just coming upon this topic can see your program developement etc
easier to see a new post and not waste bandwidth downloading the first
post to see if
changes were made

I'll make it easier, I'll put this code on github in a separate
repository so that the progress of the experiments is visible

So far, the research conducted in search of a ready-made native TLS
1.3 solution for Free Pascal or Delphi has not been crowned

It looks like we have to write everything almost from scratch

However, the free pascal repository is doing a lot of work in the
direction of cryptography, it pleases!

https://gitlab.com/freepascal.org/fpc/source/-
/tree/main/packages/gnutls - GnuTLS (Low-level binding for GnuTLS
library)

https://gitlab.com/freepascal.org/fpc/source/-/tree/main/packages/fcl-
hash
https://gitlab.com/freepascal.org/fpc/source/-/blob/main/packages/fcl-
hash/src/fphashutils.pp - Bytes transform
https://gitlab.com/freepascal.org/fpc/source/-/blob/main/packages/fcl-
hash/src/fpsha256.pp - SHA256 and HMACSha256 routines
https://gitlab.com/freepascal.org/fpc/source/-/blob/main/packages/fcl-
hash/src/fpecc.pp - Elliptic curve crypto hashing
https://gitlab.com/freepascal.org/fpc/source/-/blob/main/packages/fcl-
hash/src/fpecdsa.pp - ECDSA elliptic encryption routines
https://gitlab.com/freepascal.org/fpc/source/-/blob/main/packages/fcl-
hash/src/fptlsbigint.pas - Big Integer
https://gitlab.com/freepascal.org/fpc/source/-/blob/main/packages/fcl-
hash/src/fppem.pp - PEM key management
https://gitlab.com/freepascal.org/fpc/source/-/blob/main/packages/fcl-
hash/src/fpsha512.pp - SHA512/SHA384 and HMACSha512/HMACSha384
routines
https://gitlab.com/freepascal.org/fpc/source/-/blob/main/packages/fcl-
hash/src/fprsa.pas - RSA routines
https://gitlab.com/freepascal.org/fpc/source/-/blob/main/packages/fcl-
hash/src/onetimepass.pp - HOTP and TOTP One-time password algorithms.
Compatible with the Google Authenticator

https://gitlab.com/freepascal.org/fpc/source/-/tree/main/packages/hash
- crc, md5, sha-1, etc
https://gitlab.com/freepascal.org/fpc/source/-
/blob/main/packages/hash/src/hmac.pp - Hash-based supporting HMAC-MD5
and HMAC-SHA-1