Increase receiving capacity size (http methods, wsarecv)
lazarus / 38-Networking and Web Programming

----------------------------------------------------------------------
tudi_x · 2017-11-28 14:09

hi All,
as per the below code i am using a socket to listen for POST or GET
messages containing JSON information.
the issue that happens is that every 6 - 8 times the connection breaks
and the application that sends the information (written in go)
receives something like: Code: Pascal  [Select][+][-]window.onload =
function(){var x1 = document.getElementById("main_content_section");
if (x1) { var x = document.getElementsByClassName("geshi");for (var i
= 0; i error: Post http://127.0.0.1:55552/: net/http: HTTP/1.x
transport connection broken: read tcp 127.0.0.1:5441->127.0.0.1:55552:
wsarecv: An existing connection was forcibly closed by the remote
host.it looks like the truncation occurs after 4096 characters.

Code: Pascal  [Select][+][-]window.onload = function(){var x1 =
document.getElementById("main_content_section"); if (x1) { var x =
document.getElementsByClassName("geshi");for (var i = 0; i unit
sock_ls; {$mode objfpc}{$H+} interface uses  Classes, SysUtils,
Sockets,  blcksock,  fpjson, jsonparser,  strutils; type  TPassMessage
= procedure(AMsg: string) of object; type  TQSockListen =
class(TThread)  private    _PassMessage: TPassMessage;    
_MainSocket: word;    _IAddr: TINetSockAddr; //internet address, needs
Sockets unit    _ASocket: integer;    _RequestBody: string;    
procedure Connect;    procedure sockRead;    procedure sockWrite();  
function Rfc822DateTime(t: TDateTime): string;   protected   
procedure Execute; override;    procedure TriggerMessage(AMsg:
string);   public    constructor Create(APort: cardinal; AIP: string);
reintroduce;    destructor Destroy(); override;     property
OnPassMessage: TPassMessage read _PassMessage write _PassMessage; 
end; implementation const  packet_size = 10485760; constructor
TQSockListen.Create(APort: cardinal; AIP: string);begin  inherited
Create(False);  self.FreeOnTerminate := True;   _MainSocket :=
fpsocket(AF_INET, SOCK_STREAM, 0);  _IAddr.sin_family := AF_INET; 
_IAddr.sin_port := htons(APort);  _IAddr.sin_addr.s_addr :=
longword(StrToNetAddr(AIP));   fpbind(_MainSocket, @_IAddr,
SizeOf(_IAddr));  fplisten(_MainSocket, 1);end; procedure
TQSockListen.Execute;begin  while not Terminated do  begin   
Connect;    sockRead;     sockWrite;     {closing socket}    if
_ASocket > 0 then    begin      CloseSocket(_ASocket);    end; 
end;end; procedure TQSockListen.Connect;var  sAddrSize:
longint; begin  try    sAddrSize := SizeOf(_IAddr);    _ASocket :=
fpaccept(_MainSocket, @_IAddr, @sAddrSize);  except    on E: Exception
do      TriggerMessage('TQSocket.Connect: Error ' + E.Message); 
end;end; procedure TQSockListen.sockRead;var  buf: string;  c:
integer;  j: TJSONData;  json: string; begin  try    setLength(buf,
packet_size);    c := fprecv(_ASocket, PChar(buf), packet_size, 0);  
setLength(buf, c);     _RequestBody := buf;    
writeln(_RequestBody);    writeln('--------- end: ' +
FormatDateTime('hh:nn:ss.zzz', Now()) + '----------');    
TriggerMessage('TQSocket.sRead: request body length: ' +
IntToStr(Length(_RequestBody)) + sLineBreak + 'start|' + sLineBreak +
_RequestBody + sLineBreak + '|end');     try      {verify body has
left bracket}      if AnsiContainsStr(_RequestBody, '{') then     
begin        json := Copy(_RequestBody, Pos('{', _RequestBody),
Length(_RequestBody));         j := GetJSON(json);       
TriggerMessage('JSON parsed : ' + json);      end;     except      on
E: Exception do      begin        writeln('JSON conversion failed'); 
     TriggerMessage('position of {: ' + IntToStr(Pos('{',
_RequestBody)));        TriggerMessage('JSON conversion failed with
error: ' + E.Message);      end;    end;    FreeAndNil(j);   except  
on E: Exception do      TriggerMessage('TQSocket.sRead: Error ' +
E.Message);  end;end; procedure TQSockListen.sockWrite();var  resp:
string; begin  try    resp := 'HTTP/1.0 200' + CRLF;   
fpsend(_ASocket, PChar(resp), Length(resp), 0);     resp := 'Content-
type: Text/Html' + CRLF;    fpsend(_ASocket, PChar(resp),
Length(resp), 0);     resp := 'Content-length: 0' + CRLF;   
fpsend(_ASocket, PChar(resp), Length(resp), 0);     resp :=
'Connection: close' + CRLF;    fpsend(_ASocket, PChar(resp),
Length(resp), 0);     resp := 'Date: ' + Rfc822DateTime(now) + CRLF; 
 fpsend(_ASocket, PChar(resp), Length(resp), 0);     resp := 'Server:
Lazarus Synapse' + CRLF;    fpsend(_ASocket, PChar(resp),
Length(resp), 0);     resp := '' + CRLF;    fpsend(_ASocket,
PChar(resp), Length(resp), 0);  except    on E: Exception do     
TriggerMessage('TQSocket.sWrite: Error ' + E.Message); 
end;end; procedure TQSockListen.TriggerMessage(AMsg: string);begin  if
Assigned(_PassMessage) then    _PassMessage(AMsg);end; function
TQSockListen.Rfc822DateTime(t: TDateTime): string;var  wYear, wMonth,
wDay: word; const  MyDayNames: array[1..7] of ansistring = ('Sun',
'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat');  MyMonthNames: array[1..12]
of ansistring = ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul',
'Aug', 'Sep', 'Oct', 'Nov', 'Dec'); begin  DecodeDate(t, wYear,
wMonth, wDay);  Result := Format('%s, %d %s %s %s',
[MyDayNames[DayOfWeek(t)], wDay, MyMonthNames[wMonth],
FormatDateTime('yyyy hh":"nn":"ss', t), '+0200']);end; destructor
TQSockListen.Destroy();begin  inherited Destroy;end; end.
please advise how i could increase the capacity so this truncation
does not occur ever.
thank you!

----------------------------------------------------------------------
Remy Lebeau · 2017-11-28 21:16

Quote from: tudi_x on November 28, 2017, 03:09:21 pmthe issue that
happens is that every 6 - 8 times the connection breaks and the
application that sends the information (written in go) receives
something like: Code: Pascal  [Select][+][-]window.onload =
function(){var x1 = document.getElementById("main_content_section");
if (x1) { var x = document.getElementsByClassName("geshi");for (var i
= 0; i error: Post http://127.0.0.1:55552/: net/http: HTTP/1.x
transport connection broken: read tcp 127.0.0.1:5441->127.0.0.1:55552:
wsarecv: An existing connection was forcibly closed by the remote
host.it looks like the truncation occurs after 4096 characters.

TCP is a byte stream, there is no 1-to-1 correlation between sends and
reads.

You are using a 10MB memory buffer for reading, but fprecv() will
never be able to return that much data in a single read.  Its return
value is -1 on error, 0 on graceful disconnect, and > 0 to specify the
actual number of bytes received.  Receiving an HTTP request requires
many calls to fprecv(), so you need to call it in a loop, appending
received data to _RequestBody, until you reach the end of the HTTP
request.  Only then can you process _RequestBody correctly.

To detect the end of the HTTP request, you need to actually parse the
HTTP request as you receive it so you know when to stop reading. 
Refer to RFC 2616 Section 4.4 for the formal rules to determine the
transmission size of an HTTP message.  In a nutshell, you have to read
a CRLF-delimited line for the HTTP request line, then read CRLF-
delimited lines for the HTTP headers until you reach a CRLF CRLF
sequence denoting the end of the headers, then you have to process the
request line and headers to determine the type of message being sent
and the format of the message's body, if there is any, before you can
then read the message body according to that format.

Similarly, sending an HTTP response requires many calls to fpsend(). 
Even just sending a single string may require more than 1 call to 
fpsend(), as it can send fewer bytes than requested.  So you need to
call fpsend() in a loop as well.  Its return value is -1 on error and
> 0 to specify the actual umber of bytes sent (or more accurately, the
number of bytes accepted into the kernel buffer for eventual
transmission).  Continue calling it until there are more more bytes to
send.

You are best off not handling this manually at all.  There are plenty
of HTTP server implementations available if you look around.

----------------------------------------------------------------------
tudi_x · 2017-11-29 08:24

thank you for the clarifications. i do not intend to reinvent the
wheel.
how much data can fprecv() return? because i sent 10000 characters in
a POST request with the current code and did not have any issues fully
reading it even without a for loop.

if content length is below 200 i do not have an issue and receive it
full:
Code: Pascal  [Select][+][-]window.onload = function(){var x1 =
document.getElementById("main_content_section"); if (x1) { var x =
document.getElementsByClassName("geshi");for (var i = 0; i POST /
HTTP/1.1Host: 127.0.0.1:55552User-Agent: Go-http-client/1.1Content-
Length: 51Content-Type: application/jsonAccept-Encoding:
gzip {"request":"http://127.0.0.1:55556/v1/sube/status"}
it looks like fprecv is not the one causing the issue but maybe
something underneath.


in regards to looking around, i looked around and with my beginner
skills found:
- nYume server uses about the same code as the below on receive
- Synapse web server has in wiki a light server example with request
body read incomplete and opened
http://forum.lazarus.freepascal.org/index.php/topic,39088.0.html .
- standard library as per TFPHTTPServer has an issue when shutting
down. is reported in
https://bugs.freepascal.org/view.php?id=32727
- Indy is quite big but will consider as last option

are you referring to other HTTP implementations then the above? if
yes, could you please point me with a URL?
if not i see only option is to adapt the part that works from standard
library for reading a body which is a pity because i think in 2017
HTTP issues related to the standard library should not arise

----------------------------------------------------------------------
Remy Lebeau · 2017-11-29 18:09

Quote from: tudi_x on November 29, 2017, 09:24:19 ami do not intend to
reinvent the wheel.

Then don't.  Like I said, there are plenty of existing HTTP server
implementations available.

Quote from: tudi_x on November 29, 2017, 09:24:19 amhow much data can
fprecv() return?

That entirely depends on:

- the size of the kernel receive buffer that backs the socket

- the size of the memory buffer that you receive into.

You can customize the size of the kernel buffer using setsockopt()
with the SO_RCVBUF option, and use getsockopt() to retrieve the actual
buffer size (setsockopt() might not assign the exact size you
specify).

But in any case, it doesn't really matter what the buffer size is. 
You still need to be prepared to call fprecv() in a loop, and you MUST
parse the HTTP message data to detect end-of-message correctly.

Quote from: tudi_x on November 29, 2017, 09:24:19 ambecause i sent
10000 characters in a POST request with the current code and did not
have any issues fully reading it even without a for loop.

You still need to be prepared to call fprecv() multiple times, if for
no other reason than you have to stop reading when you reach the end
of the HTTP headers, because they tell you how to read the rest of the
data, so you would have to adjust your subsequent reading logic
accordingly until you reach the end of the message.

Quote from: tudi_x on November 29, 2017, 09:24:19 amif content length
is below 200 i do not have an issue and receive it full:

You can't rely on that behavior.  Again, TCP is a streaming transport,
and fprecv() can return fewer bytes than requested.  The contract
between you and fprecv() is that it can return as few as 1 byte, and
at most the number of bytes you request.  Which means it can return
anything in between.  You MUST account for that possibility.  fprecv()
returns whatever is *currently* in the kernel buffer, or if the buffer
is empty then it waits for at least 1 byte to arrive (if using a
blocking socket) and then returns whatever it can.

Quote from: tudi_x on November 29, 2017, 09:24:19 amit looks like
fprecv is not the one causing the issue but maybe something
underneath.

Yeah - YOU.  You are not reading and processing the HTTP data
correctly.  HTTP is a structured protocol, but you are not following
the structure at all.

Quote from: tudi_x on November 29, 2017, 09:24:19 amare you referring
to other HTTP implementations then the above?

I'm biased towards Indy (but only because I develop it!).  It has a
working TIdHTTPServer component.

I'm sure other HTTP server implementations also work.

----------------------------------------------------------------------
tudi_x · 2017-11-30 11:45

thank you!
based on your feedback i managed the below which serves my purpose of
receiving JSON for now.

Code: Pascal  [Select][+][-]window.onload = function(){var x1 =
document.getElementById("main_content_section"); if (x1) { var x =
document.getElementsByClassName("geshi");for (var i = 0; i
{https://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html}      {message
type}      message.Append(ASocket.RecvString(Timeout));      {$IF
DEFINED(Debug)}      WriteLn(message.Strings[message.Count - 1]);    
{$EndIF}       {headers}      repeat       
message.Append(ASocket.RecvString(Timeout));       
WriteLn(message.Strings[message.Count - 1]);         if
AnsiContainsText(message.Strings[message.Count - 1], 'Content-Length')
then        begin          content :=
StrToInt(copy(message.Strings[message.Count - 1], pos(':',
message.Strings[message.Count - 1]) + 1,
Length(message.Strings[message.Count - 1])));        end;      until
message.Strings[message.Count - 1] = '';       {body}      if content
> 0 then      begin        without_body := Length(message.Text);     
 writeln('content: ' + IntToStr(content) + '|' + 'headers: ' +
IntToStr(without_body));        TriggerMessage('content: ' +
IntToStr(content) + '|' + 'headers: ' + IntToStr(without_body));     
  repeat          message.Append(ASocket.RecvString(Timeout));       
  {$IF DEFINED(Debug)}          writeln('all: ' +
IntToStr(length(message.Text)));         
WriteLn(message.Strings[message.Count - 1]);         
writeln(IntToStr(length(message.Text) - without_body - content));    
    {$EndIF}         until length(message.Text) - without_body >=
content;      end;       TriggerMessage(message.Text);     
message.Clear;

----------------------------------------------------------------------
Remy Lebeau · 2017-11-30 19:47

Quote from: tudi_x on November 30, 2017, 12:45:39 pmbased on your
feedback i managed the below which serves my purpose of receiving JSON
for now.

That code is very wrong, and will likely break eventually.

When reading the request line and headers, you MUST read CRLF-
delimited strings, which a function like RecvString() is likely NOT
doing.

You are not accounting for the possibility that the 'Content-Length'
header may not always be present (for instance 'Transfer-Encoding:
chunked' may be used instead), or the fact that not all message types
even have a message body to begin with, even if there is a 'Content-
Length' present.

And even if there is a message body to read, you are not taking the
'Content-Length' value into account correctly when reading the body. 
You can't just blindly read arbitrary strings until you reach the
desired size, you might read too much data (such as if the client is
using HTTP pipelining, for instance).  You have to read EXACTLY how
much 'Content-Length' says to read, no more, no less.  And in the case
that something other than 'Content-Length' is used to denote the
message size, you are not reading the message body at all.

In short, you are not following the rules of RFC 2616 Section 4.4 (and
its successor, RFC 7230 Section 3.3.3).  Until you do, your code is
risking corrupting the HTTP communications.