Rest API chunk file download
lazarus / 21-General

----------------------------------------------------------------------
daniel_sap · 2025-03-24 15:47

Hi,
I'm writing a rest api endpoint which should return a file chunk by
chunk.
I wrote some code which tries to do it

Code: Pascal  [Select][+][-]window.onload = function(){var x1 =
document.getElementById("main_content_section"); if (x1) { var x =
document.getElementsByClassName("geshi");for (var i = 0; i program
ChunkDownload; {$mode objfpc}{$H+} uses  {$IFDEF UNIX}  cthreads, 
{$ENDIF}  SysUtils, fphttpapp, httproute, HTTPDefs,
Classes;  procedure downloadFileWithChunks(aRequest: TRequest;
aResponse: TResponse);const  CHUNK_SIZE = 8192; // 8 KB chunksvar 
FilePath: string;  FileStream: TFileStream;  Buffer:
array[0..CHUNK_SIZE-1] of Byte;  BytesRead: Integer;  CustomFileName:
string;  TempStream: TMemoryStream;begin  FilePath :=
'D:\SomeBigFile.exe';   if FilePath = '' then  begin   
aResponse.Content := '{"result": "Missing file path"}';   
aResponse.Code := 400;    aResponse.ContentLength :=
Length(aResponse.Content);    aResponse.SetCustomHeader('Access-
Control-Allow-Origin', '*');    aResponse.SendContent;   
WriteLn('Sent 400 response');    Exit;  end;   if not
FileExists(FilePath) then  begin    aResponse.Content := '{"result":
"File not found: ' + FilePath + '"}';    aResponse.Code := 404;   
aResponse.ContentLength := Length(aResponse.Content);   
aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');   
aResponse.SendContent;    WriteLn('Sent 404 response for path: ',
FilePath);    Exit;  end;   try    FileStream :=
TFileStream.Create(FilePath, fmOpenRead or fmShareDenyWrite);    try 
   CustomFileName := ExtractFileName(FilePath);       // Set headers 
   aResponse.ContentType := 'application/x-msdownload';     
aResponse.ContentLength := FileStream.Size;     
aResponse.SetCustomHeader('Content-Disposition', 'attachment;
filename="' + CustomFileName + '"');     
aResponse.SetCustomHeader('Accept-Ranges', 'bytes');     
aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');      
WriteLn('Headers set:');      WriteLn('Content-Type: ',
aResponse.ContentType);      WriteLn('Content-Length: ',
IntToStr(aResponse.ContentLength));      WriteLn('Content-Disposition:
attachment; filename="', CustomFileName, '"');      WriteLn('Access-
Control-Allow-Origin: *');       // Send headers     
aResponse.SendResponse;      WriteLn('Headers sent via
SendResponse');       // Prepare and send content       TempStream :=
TMemoryStream.Create;      aResponse.ContentStream := TempStream;    
try        while FileStream.Position < FileStream.Size do       
begin//          BytesRead := FileStream.Read(Buffer, CHUNK_SIZE);   
     TempStream.Position := 0;          BytesRead :=
TempStream.CopyFrom(FileStream, CHUNK_SIZE);           if BytesRead <=
0 then            break;           TempStream.Position := 0;         
aResponse.SendContent;        end;      finally       
TempStream.Free; // Framework takes ownership after assignment     
end;    finally      FileStream.Free;    end;  except    on E:
Exception do    begin      aResponse.Content := '{"result": "Error: '
+ E.ClassName + ' - ' + E.Message + '"}';      aResponse.Code := 500; 
    aResponse.ContentLength := Length(aResponse.Content);     
aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');     
aResponse.SendContent;      WriteLn('Error: ', E.ClassName, ' - ',
E.Message);    end;  end; end; begin  Application.Port := 9090;  
HTTPRouter.RegisterRoute('/file', TRouteMethod.rmGet,
@downloadFileWithChunks);   Application.Threaded := true; 
Application.Initialize();   WriteLn('Server starter at port: ' +
IntToStr(Application.Port));  Application.Run;end. 
It's not working cause after the SendResponse content is allready sent
and SendContent causes an error.
May be you can advice how to properly send chunk HTTP Response

Here is also the code of the HTML file which triggers the download

Code: Text  [Select][+][-]window.onload = function(){var x1 =
document.getElementById("main_content_section"); if (x1) { var x =
document.getElementsByClassName("geshi");for (var i = 0; i <!DOCTYPE
html><html lang="en"><head>  <meta charset="UTF-8" />  <meta
name="viewport" content="width=device-width, initial-scale=1.0" /> 
<title>Download File in Chunks</title></head><body>  <h1>Download File
from REST API</h1>  <label for="filePath">Enter File Path:</label> 
<input type="text" id="filePath" placeholder="/path/to/file.txt" /> 
<button onclick="downloadFile()">Download File</button>   <script>   
async function downloadFile() {      const filePath =
document.getElementById('filePath').value;      if (!filePath) {     
 alert('Please enter a valid file path');        return;      }      
const downloadUrl =
`http://localhost:9090/file?path=${encodeURIComponent(filePath)}`;   
  try {        const response = await fetch(downloadUrl);        if
(!response.ok) {          throw new Error(`Server returned
${response.status}: ${await response.text()}`);        }         //
Log all response headers        console.log('Response headers:');    
  for (const [key, value] of response.headers) {         
console.log(`${key}: ${value}`);        }         const totalSize =
parseInt(response.headers.get('Content-Length'), 10);        let
receivedSize = 0;         // Extract filename from Content-
Disposition        let fileName = 'downloaded_file';        const
disposition = response.headers.get('Content-Disposition');        if
(disposition && disposition.includes('filename=')) {          const
matches = disposition.match(/filename="([^"]+)"/);          if
(matches && matches[1]) {            fileName = matches[1];         
}        }        console.log('Using filename:', fileName);        
const reader = response.body.getReader();        const chunks = [];  
            while (true) {          const { done, value } = await
reader.read();          if (done) {                   
console.log('done');                        break;                 
}           chunks.push(value);          receivedSize +=
value.length;          console.log(`Received ${receivedSize} of
${totalSize} bytes`);        }         const blob = new Blob(chunks); 
      const url = window.URL.createObjectURL(blob);        const link
= document.createElement('a');        link.href = url;       
link.download = fileName;        document.body.appendChild(link);    
  link.click();        document.body.removeChild(link);       
window.URL.revokeObjectURL(url);         console.log('Download
completed successfully');      } catch (error) {       
console.error('Download failed:', error);        alert('Download
failed: ' + error.message);      }    }  </script></body></html>

----------------------------------------------------------------------
Leledumbo · 2025-03-24 22:34

Chunked transfer encoding is not yet implemented, every TResponse
instance is a one time use by calling SendContent.

There is a way, but it breaks TResponse abstraction. Since you're
using fphttpapp, the backend server is fphttpserver. You can downcast
TResponse to TFPHTTPConnectionResponse then instead of calling
AResponse.SendContent, call AResponse.Connection.Socket.WriteBuffer()
directly in a loop, where each iteration is a chunk. Don't forget to
send the correct headers beforehand, as you must also follow the
chunked transfer encoding protocol. At the most basic, you forgot to
set:
Code: [Select]Transfer-Encoding: chunkedYour code also doesn't follow
the chunk format, which should be:
Code: [Select]<chunk size>CRLF<chunk data>CRLFfinally terminated by:
Code: [Select]0CRLFCRLF

----------------------------------------------------------------------
daniel_sap · 2025-03-25 06:49

Thank you, Leledumbo, for providing everything needed for
understanding and implementing

I made the changes and here is the working code
Code: Pascal  [Select][+][-]window.onload = function(){var x1 =
document.getElementById("main_content_section"); if (x1) { var x =
document.getElementsByClassName("geshi");for (var i = 0; i program
ChunkDownload; {$mode objfpc}{$H+} uses  {$IFDEF UNIX}  cthreads, 
{$ENDIF}  SysUtils, fphttpapp, httproute, HTTPDefs, Classes,
fphttpserver, fpWeb;  type   TFPHTTPConnectionResponseHack =
class(TFPHTTPConnectionResponse); procedure
downloadFileWithChunks(aRequest: TRequest; aResponse:
TResponse);const  CHUNK_SIZE = 8192; // 8 KB chunksvar  FilePath:
string;  FileStream: TFileStream;  Buffer: array of Byte;  BytesRead:
Integer;  CustomFileName: string;  connection: TFPHTTPConnection; 
ChunkHeader: string;begin  if not (aResponse is
TFPHTTPConnectionResponse) then    raise Exception.Create('Invalid
response type, not TFPHTTPConnectionResponse');   if
TFPHTTPConnectionResponseHack(aResponse).Connection = nil then   
raise Exception.Create('Connection is nil. Ensure the request is
handled via fphttpserver.');   FilePath := 'D:\Project\Lazarus\Applica
tions\Silhouette\SilSetup\windows\x86_64-
win64\Output\Silhouette_6_x86_64-win64.exe';   if FilePath = '' then 
begin    aResponse.Content := '{"result": "Missing file path"}';   
aResponse.Code := 400;    aResponse.ContentLength :=
Length(aResponse.Content);    aResponse.SetCustomHeader('Access-
Control-Allow-Origin', '*');    aResponse.SendContent;   
WriteLn('Sent 400 response');    Exit;  end;   if not
FileExists(FilePath) then  begin    aResponse.Content := '{"result":
"File not found: ' + FilePath + '"}';    aResponse.Code := 404;   
aResponse.ContentLength := Length(aResponse.Content);   
aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');   
aResponse.SendContent;    WriteLn('Sent 404 response for path: ',
FilePath);    Exit;  end;   try    FileStream :=
TFileStream.Create(FilePath, fmOpenRead or fmShareDenyWrite);    try 
   connection := TFPHTTPConnectionResponseHack(aResponse).Connection; 
    CustomFileName := ExtractFileName(FilePath);       // Set headers 
    aResponse.ContentType := 'application/x-msdownload';     
aResponse.ContentLength := FileStream.Size;     
aResponse.SetCustomHeader('Content-Disposition', 'attachment;
filename="' + CustomFileName + '"');     
aResponse.SetCustomHeader('Accept-Ranges', 'bytes');     
aResponse.SetCustomHeader('Access-Control-Allow-Origin', '*');     
AResponse.SetCustomHeader('Access-Control-Expose-Headers', 'Content-
Disposition, Content-Length, Transfer-Encoding');     
aResponse.SetCustomHeader('Transfer-Encoding', 'chunked');     
TFPHTTPConnectionResponse(aResponse).SendHeaders;      
WriteLn('Headers set:');      WriteLn('Content-Type: ',
aResponse.ContentType);      WriteLn('Content-Length: ',
IntToStr(aResponse.ContentLength));      WriteLn('Content-Disposition:
attachment; filename="', CustomFileName, '"');      WriteLn('Access-
Control-Allow-Origin: *');      // Allocate buffer   
 SetLength(Buffer, CHUNK_SIZE);      while FileStream.Position <
FileStream.Size do      begin        BytesRead :=
FileStream.Read(Buffer[0], CHUNK_SIZE);        if BytesRead <= 0 then 
        break;         // Write chunk size in hexadecimal followed by
CRLF        ChunkHeader := IntToHex(BytesRead, 1) + #13#10;       
Connection.Socket.WriteBuffer(ChunkHeader[1], Length(ChunkHeader));  
     // Write the actual data followed by CRLF       
Connection.Socket.WriteBuffer(Buffer[0], BytesRead);       
Connection.Socket.WriteBuffer(#13#10, 2);      end;       // Send
final chunk (0 length) to indicate end of transfer      ChunkHeader :=
'0'#13#10#13#10;      Connection.Socket.WriteBuffer(ChunkHeader[1],
Length(ChunkHeader));    finally      FileStream.Free;    end;  
except    on E: Exception do    begin      aResponse.Content :=
'{"result": "Error: ' + E.ClassName + ' - ' + E.Message + '"}';     
aResponse.Code := 500;      aResponse.ContentLength :=
Length(aResponse.Content);      aResponse.SetCustomHeader('Access-
Control-Allow-Origin', '*');      aResponse.SendContent;     
WriteLn('Error: ', E.ClassName, ' - ', E.Message);    end; 
end; end; begin  Application.Port := 9090;  
HTTPRouter.RegisterRoute('/file', TRouteMethod.rmGet,
@downloadFileWithChunks);   Application.Threaded := true; 
Application.Initialize();   WriteLn('Server starter at port: ' +
IntToStr(Application.Port));  Application.Run;end. 
I had to introduce TFPHTTPConnectionResponseHack class
cause Connection property is not accessible in
TFPHTTPConnectionResponse
and it's parent introduces property Connection: String

----------------------------------------------------------------------
Leledumbo · 2025-03-26 00:01

Quote from: daniel_sap on March 25, 2025, 07:49:00 amI had to
introduce TFPHTTPConnectionResponseHack class
cause Connection property is not accessible in
TFPHTTPConnectionResponse
and it's parent introduces property Connection: String
Ah yes, I forgot its visibility is protected. Despite working, I guess
the protected visibility is broken here since you didn't reintroduce
the property with higher visibility. Compiler maintainers should be
able to ensure, hope they see this.