Fix: node to use chunking for uploads

src/SDK/Language/Node.php

@@ -74,7 +74,7 @@ public function getTypeName(array $parameter, array $method = []): string
                 }
                 return 'any[]';
             case self::TYPE_FILE:
-                return "File";
+                return "File | InputFile";
             case self::TYPE_OBJECT:
                 if (empty($method)) {
                     return $parameter['type'];
@@ -186,6 +186,36 @@ public function getParamExample(array $param, string $lang = ''): string
         };
     }
 
+    /**
+     * Check if service has any file parameters
+     *
+     * @param array $service
+     * @return bool
+     */
+    public function hasFileParam(array $service): bool
+    {
+        foreach ($service['methods'] as $method) {
+            foreach ($method['parameters']['all'] as $parameter) {
+                if ($parameter['type'] === self::TYPE_FILE) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    /**
+     * @return array
+     */
+    public function getFilters(): array
+    {
+        return \array_merge(parent::getFilters(), [
+            new \Twig\TwigFilter('hasFileParam', function ($service) {
+                return $this->hasFileParam($service);
+            }),
+        ]);
+    }
+
     /**
      * @return array
      */

templates/node/src/client.ts.twig

@@ -1,6 +1,7 @@
 import { fetch, FormData, File } from 'node-fetch-native-with-agent';
 import { createAgent } from 'node-fetch-native-with-agent/agent';
 import { Models } from './models';
+import { InputFile } from './inputFile';
 
 type Payload = {
     [key: string]: any;
@@ -205,12 +206,60 @@ class Client {
     }
 
     async chunkedUpload(method: string, url: URL, headers: Headers = {}, originalPayload: Payload = {}, onProgress: (progress: UploadProgress) => void) {
-        const [fileParam, file] = Object.entries(originalPayload).find(([_, value]) => value instanceof File) ?? [];
+        const [fileParam, file] = Object.entries(originalPayload).find(
+            ([_, value]) => value instanceof File || value instanceof InputFile
+        ) ?? [];
 
         if (!file || !fileParam) {
             throw new Error('File not found in payload');
         }
 
+        if (file instanceof InputFile) {
+            const size = await file.size();
+
+            if (size <= Client.CHUNK_SIZE) {
+                const payload = { ...originalPayload };
+                payload[fileParam] = await file.toFile();
+                return await this.call(method, url, headers, payload);
+            }
+
+            let start = 0;
+            let response = null;
+
+            while (start < size) {
+                let end = start + Client.CHUNK_SIZE;
+                if (end >= size) {
+                    end = size;
+                }
+
+                headers['content-range'] = `bytes ${start}-${end - 1}/${size}`;
+                const chunk = await file.slice(start, end);
+
+                const payload = { ...originalPayload };
+                payload[fileParam] = new File([chunk], file.filename);
+
+                response = await this.call(method, url, headers, payload);
+
+                if (onProgress && typeof onProgress === 'function') {
+                    onProgress({
+                        $id: response.$id,
+                        progress: Math.round((end / size) * 100),
+                        sizeUploaded: end,
+                        chunksTotal: Math.ceil(size / Client.CHUNK_SIZE),
+                        chunksUploaded: Math.ceil(end / Client.CHUNK_SIZE)
+                    });
+                }
+
+                if (response && response.$id) {
+                    headers['x-{{spec.title | caseLower }}-id'] = response.$id;
+                }
+
+                start = end;
+            }
+
+            return response;
+        }
+
         if (file.size <= Client.CHUNK_SIZE) {
             return await this.call(method, url, headers, originalPayload);
         }

templates/node/src/inputFile.ts.twig

@@ -1,23 +1,110 @@
 import { File } from "node-fetch-native-with-agent";
-import { realpathSync, readFileSync } from "fs";
-import type { BinaryLike } from "crypto";
+import { basename } from "path";
+import { realpathSync } from "fs";
+import { promises as fs } from "fs";
+type BlobLike = {
+  size: number;
+  slice: (start: number, end: number) => BlobLike;
+  arrayBuffer: () => Promise<ArrayBuffer>;
+};
+
+type InputFileSource =
+  | { type: 'path'; path: string }
+  | { type: 'buffer'; data: Buffer }
+  | { type: 'blob'; data: BlobLike };
 
 export class InputFile {
-  static fromBuffer(
-    parts: Blob | BinaryLike,
-    name: string
-  ): File {
-    return new File([parts], name);
+  private source: InputFileSource;
+  filename: string;
+
+  private constructor(source: InputFileSource, filename: string) {
+    this.source = source;
+    this.filename = filename;
+  }
+
+  static fromBuffer(parts: BlobLike | Buffer | Uint8Array | ArrayBuffer | string, name: string): InputFile {
+    if (parts && typeof (parts as BlobLike).arrayBuffer === 'function') {
+      return new InputFile({ type: 'blob', data: parts as BlobLike }, name);
+    }
+
+    if (Buffer.isBuffer(parts)) {
+      return new InputFile({ type: 'buffer', data: parts }, name);
+    }
+
+    if (typeof parts === 'string') {
+      return new InputFile({ type: 'buffer', data: Buffer.from(parts) }, name);
+    }
+
+    if (parts instanceof ArrayBuffer) {
+      return new InputFile({ type: 'buffer', data: Buffer.from(parts) }, name);
+    }
+
+    if (ArrayBuffer.isView(parts)) {
+      return new InputFile({
+        type: 'buffer',
+        data: Buffer.from(parts.buffer, parts.byteOffset, parts.byteLength),
+      }, name);
+    }
+
+    throw new Error('Unsupported input type for InputFile.fromBuffer');
   }
 
-  static fromPath(path: string, name: string): File {
+  static fromPath(path: string, name?: string): InputFile {
     const realPath = realpathSync(path);
-    const contents = readFileSync(realPath);
-    return this.fromBuffer(contents, name);
+    return new InputFile({ type: 'path', path: realPath }, name ?? basename(realPath));
+  }
+
+  static fromPlainText(content: string, name: string): InputFile {
+    return new InputFile({ type: 'buffer', data: Buffer.from(content) }, name);
+  }
+
+  async size(): Promise<number> {
+    switch (this.source.type) {
+      case 'path':
+        return (await fs.stat(this.source.path)).size;
+      case 'buffer':
+        return this.source.data.length;
+      case 'blob':
+        return this.source.data.size;
+    }
+  }
+
+  async slice(start: number, end: number): Promise<Buffer> {
+    const length = end - start;
+
+    switch (this.source.type) {
+      case 'path': {
+        const handle = await fs.open(this.source.path, 'r');
+        try {
+          const buffer = Buffer.alloc(length);
+          const result = await handle.read(buffer, 0, length, start);
+          return result.bytesRead === buffer.length ? buffer : buffer.subarray(0, result.bytesRead);
+        } finally {
+          await handle.close();
+        }
+      }
+      case 'buffer':
+        return this.source.data.subarray(start, end);
+      case 'blob': {
+        const arrayBuffer = await this.source.data.slice(start, end).arrayBuffer();
+        return Buffer.from(arrayBuffer);
+      }
+    }
+  }
+
+  async toFile(): Promise<File> {
+    const buffer = await this.toBuffer();
+    return new File([buffer], this.filename);
   }
 
-  static fromPlainText(content: string, name: string): File {
-    const arrayBytes = new TextEncoder().encode(content);
-    return this.fromBuffer(arrayBytes, name);
+  private async toBuffer(): Promise<Buffer> {
+    switch (this.source.type) {
+      case 'path':
+        return await fs.readFile(this.source.path);
+      case 'buffer':
+        return this.source.data;
+      case 'blob':
+        return Buffer.from(await this.source.data.arrayBuffer());
+    }
   }
 }

templates/node/src/services/template.ts.twig

@@ -1,6 +1,10 @@
 import { {{ spec.title | caseUcfirst}}Exception, Client, type Payload, UploadProgress } from '../client';
 import type { Models } from '../models';
 
+{% if service | hasFileParam %}
+import { InputFile } from '../inputFile';
+{% endif %}
+
 {% set added = [] %}
 {% for method in service.methods %}
 {% for parameter in method.parameters.all %}

tests/Node16Test.php

@@ -27,7 +27,10 @@ class Node16Test extends Base
         ...Base::BAR_RESPONSES, // Object params
         ...Base::GENERAL_RESPONSES,
         ...Base::UPLOAD_RESPONSES,
-        ...Base::UPLOAD_RESPONSES, // Object params
+        ...Base::LARGE_FILE_RESPONSES,
+        ...Base::LARGE_FILE_RESPONSES,
+        ...Base::LARGE_FILE_RESPONSES,
+        ...Base::LARGE_FILE_RESPONSES, // Large file uploads
         ...Base::ENUM_RESPONSES,
         ...Base::MODEL_RESPONSES,
         ...Base::EXCEPTION_RESPONSES,

tests/Node18Test.php

@@ -27,7 +27,10 @@ class Node18Test extends Base
         ...Base::BAR_RESPONSES, // Object params
         ...Base::GENERAL_RESPONSES,
         ...Base::UPLOAD_RESPONSES,
-        ...Base::UPLOAD_RESPONSES, // Object params
+        ...Base::LARGE_FILE_RESPONSES,
+        ...Base::LARGE_FILE_RESPONSES,
+        ...Base::LARGE_FILE_RESPONSES,
+        ...Base::LARGE_FILE_RESPONSES, // Large file uploads
         ...Base::ENUM_RESPONSES,
         ...Base::MODEL_RESPONSES,
         ...Base::EXCEPTION_RESPONSES,

tests/Node20Test.php

@@ -27,7 +27,10 @@ class Node20Test extends Base
         ...Base::BAR_RESPONSES, // Object params
         ...Base::GENERAL_RESPONSES,
         ...Base::UPLOAD_RESPONSES,
-        ...Base::UPLOAD_RESPONSES, // Object params
+        ...Base::LARGE_FILE_RESPONSES,
+        ...Base::LARGE_FILE_RESPONSES,
+        ...Base::LARGE_FILE_RESPONSES,
+        ...Base::LARGE_FILE_RESPONSES, // Large file uploads
         ...Base::ENUM_RESPONSES,
         ...Base::MODEL_RESPONSES,
         ...Base::EXCEPTION_RESPONSES,