Skip to content

Commit f3b9204

Browse files
AllieJonssonAllieJonsson
authored
fix(ndjson): fix issues with streaming ndjson (#2167)
* fix(ndjson): fix issues with streaming ndjson * fix unused import error
1 parent 3ae2111 commit f3b9204

File tree

12 files changed

+333
-84
lines changed

12 files changed

+333
-84
lines changed

‎docs/public/sitemap.xml‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -7,141 +7,141 @@
77

88
<url>
99
<loc>https://orval.dev/guides/angular</loc>
10-
<lastmod>2025-04-20</lastmod>
10+
<lastmod>2025-06-16</lastmod>
1111
</url>
1212

1313
<url>
1414
<loc>https://orval.dev/guides/basics</loc>
15-
<lastmod>2025-04-20</lastmod>
15+
<lastmod>2025-06-16</lastmod>
1616
</url>
1717

1818
<url>
1919
<loc>https://orval.dev/guides/client-with-zod</loc>
20-
<lastmod>2025-04-20</lastmod>
20+
<lastmod>2025-06-16</lastmod>
2121
</url>
2222

2323
<url>
2424
<loc>https://orval.dev/guides/custom-axios</loc>
25-
<lastmod>2025-04-20</lastmod>
25+
<lastmod>2025-06-16</lastmod>
2626
</url>
2727

2828
<url>
2929
<loc>https://orval.dev/guides/custom-client</loc>
30-
<lastmod>2025-04-20</lastmod>
30+
<lastmod>2025-06-16</lastmod>
3131
</url>
3232

3333
<url>
3434
<loc>https://orval.dev/guides/enums</loc>
35-
<lastmod>2025-04-20</lastmod>
35+
<lastmod>2025-06-16</lastmod>
3636
</url>
3737

3838
<url>
3939
<loc>https://orval.dev/guides/fetch-client</loc>
40-
<lastmod>2025-04-20</lastmod>
40+
<lastmod>2025-06-16</lastmod>
4141
</url>
4242

4343
<url>
4444
<loc>https://orval.dev/guides/fetch</loc>
45-
<lastmod>2025-04-20</lastmod>
45+
<lastmod>2025-06-16</lastmod>
4646
</url>
4747

4848
<url>
4949
<loc>https://orval.dev/guides/hono</loc>
50-
<lastmod>2025-04-20</lastmod>
51-
</url>
52-
53-
<url>
54-
<loc>https://orval.dev/guides/mcp-blog-ja</loc>
55-
<lastmod>2025-04-20</lastmod>
50+
<lastmod>2025-06-16</lastmod>
5651
</url>
5752

5853
<url>
5954
<loc>https://orval.dev/guides/mcp</loc>
60-
<lastmod>2025-04-20</lastmod>
55+
<lastmod>2025-06-16</lastmod>
6156
</url>
6257

6358
<url>
6459
<loc>https://orval.dev/guides/msw</loc>
65-
<lastmod>2025-04-20</lastmod>
60+
<lastmod>2025-06-16</lastmod>
6661
</url>
6762

6863
<url>
6964
<loc>https://orval.dev/guides/react-query</loc>
70-
<lastmod>2025-04-20</lastmod>
65+
<lastmod>2025-06-16</lastmod>
7166
</url>
7267

7368
<url>
7469
<loc>https://orval.dev/guides/set-base-url</loc>
75-
<lastmod>2025-04-20</lastmod>
70+
<lastmod>2025-06-16</lastmod>
71+
</url>
72+
73+
<url>
74+
<loc>https://orval.dev/guides/stream-ndjson</loc>
75+
<lastmod>2025-06-16</lastmod>
7676
</url>
7777

7878
<url>
7979
<loc>https://orval.dev/guides/svelte-query</loc>
80-
<lastmod>2025-04-20</lastmod>
80+
<lastmod>2025-06-16</lastmod>
8181
</url>
8282

8383
<url>
8484
<loc>https://orval.dev/guides/swr</loc>
85-
<lastmod>2025-04-20</lastmod>
85+
<lastmod>2025-06-16</lastmod>
8686
</url>
8787

8888
<url>
8989
<loc>https://orval.dev/guides/vue-query</loc>
90-
<lastmod>2025-04-20</lastmod>
90+
<lastmod>2025-06-16</lastmod>
9191
</url>
9292

9393
<url>
9494
<loc>https://orval.dev/guides/zod</loc>
95-
<lastmod>2025-04-20</lastmod>
95+
<lastmod>2025-06-16</lastmod>
9696
</url>
9797

9898
<url>
9999
<loc>https://orval.dev/installation</loc>
100-
<lastmod>2025-04-20</lastmod>
100+
<lastmod>2025-06-16</lastmod>
101101
</url>
102102

103103
<url>
104104
<loc>https://orval.dev/overview</loc>
105-
<lastmod>2025-04-20</lastmod>
105+
<lastmod>2025-06-16</lastmod>
106106
</url>
107107

108108
<url>
109109
<loc>https://orval.dev/quick-start</loc>
110-
<lastmod>2025-04-20</lastmod>
110+
<lastmod>2025-06-16</lastmod>
111111
</url>
112112

113113
<url>
114114
<loc>https://orval.dev/reference/cli</loc>
115-
<lastmod>2025-04-20</lastmod>
115+
<lastmod>2025-06-16</lastmod>
116116
</url>
117117

118118
<url>
119119
<loc>https://orval.dev/reference/configuration/full-example</loc>
120-
<lastmod>2025-04-20</lastmod>
120+
<lastmod>2025-06-16</lastmod>
121121
</url>
122122

123123
<url>
124124
<loc>https://orval.dev/reference/configuration/hooks</loc>
125-
<lastmod>2025-04-20</lastmod>
125+
<lastmod>2025-06-16</lastmod>
126126
</url>
127127

128128
<url>
129129
<loc>https://orval.dev/reference/configuration/input</loc>
130-
<lastmod>2025-04-20</lastmod>
130+
<lastmod>2025-06-16</lastmod>
131131
</url>
132132

133133
<url>
134134
<loc>https://orval.dev/reference/configuration/output</loc>
135-
<lastmod>2025-04-20</lastmod>
135+
<lastmod>2025-06-16</lastmod>
136136
</url>
137137

138138
<url>
139139
<loc>https://orval.dev/reference/configuration/overview</loc>
140-
<lastmod>2025-04-20</lastmod>
140+
<lastmod>2025-06-16</lastmod>
141141
</url>
142142

143143
<url>
144144
<loc>https://orval.dev/reference/integration</loc>
145-
<lastmod>2025-04-20</lastmod>
145+
<lastmod>2025-06-16</lastmod>
146146
</url>
147147
</urlset>

‎docs/src/manifests/manifest.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,11 @@
109109
"title": "MCP server",
110110
"path": "/guides/mcp",
111111
"editUrl": "/guides/mcp.md"
112+
},
113+
{
114+
"title": "Stream NDJSON",
115+
"path": "/guides/stream-ndjson",
116+
"editUrl": "/guides/stream-ndjson.md"
112117
}
113118
]
114119
},

‎docs/src/pages/guides/stream-ndjson.md‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
---
2+
id: stream-ndjson
3+
title: Stream Newline Delimited JSON
4+
---
5+
6+
### Introduction
7+
8+
`Orval` generates code that properly types responses streamed from `NDJSON`.
9+
[NDJSON](https://en.wikipedia.org/wiki/JSON_streaming#Newline-delimited_JSON) is a technique to stream an array of JSON objects. This is mostly used when the data set is large.
10+
11+
### How to use
12+
13+
`Orval` does not generate code for actually parsing the stream, but rather provides type safety. You can use the code in the example below to see how you can achieve reading streamed data.
14+
Proper type support is only supported when using the `fetch` client as either a standalone client, or as a [httpClient](../reference/configuration/output#httpclient).
15+
16+
#### Example
17+
18+
```ts
19+
// orval.config.ts
20+
import { defineConfig } from 'orval';
21+
22+
export default defineConfig({
23+
petstore: {
24+
input: {
25+
target: './stream.yaml',
26+
},
27+
output: {
28+
client: 'fetch',
29+
target: 'src/endpoints.ts',
30+
schemas: 'src/model',
31+
},
32+
},
33+
});
34+
```
35+
36+
```yml
37+
openapi: 3.1.0
38+
info:
39+
version: 1.0.0
40+
title: Stream
41+
paths:
42+
/stream:
43+
get:
44+
operationId: stream
45+
description: Stream results
46+
responses:
47+
'200':
48+
description: The stream result.
49+
content:
50+
application/x-ndjson:
51+
schema:
52+
$ref: '#/components/schemas/StreamEntry'
53+
components:
54+
schemas:
55+
StreamEntry:
56+
type: object
57+
properties:
58+
foo:
59+
type: number
60+
bar:
61+
type: string
62+
```
63+
64+
```ts
65+
// Generated code
66+
interface TypedResponse<T> extends Response {
67+
json(): Promise<T>;
68+
}
69+
70+
/**
71+
* Stream results
72+
*/
73+
export type streamResponse200 = {
74+
stream: TypedResponse<StreamEntry>;
75+
status: 200;
76+
};
77+
export type streamResponseComposite = streamResponse200;
78+
79+
export type streamResponse = streamResponseComposite & {
80+
headers: Headers;
81+
};
82+
83+
export const getStreamUrl = () => {
84+
return `/stream`;
85+
};
86+
87+
export const stream = async (
88+
options?: RequestInit,
89+
): Promise<streamResponse> => {
90+
const stream = await fetch(getStreamUrl(), {
91+
...options,
92+
method: 'GET',
93+
headers: { Accept: 'application/x-ndjson', ...options?.headers },
94+
});
95+
96+
return {
97+
status: stream.status,
98+
stream,
99+
headers: stream.headers,
100+
} as streamResponse;
101+
};
102+
```
103+
104+
```ts
105+
// Calling code
106+
export const readStream = <T extends object>(
107+
response: Response & { json(): Promise<T> },
108+
processLine: (value: T) => void | boolean,
109+
onError?: (response?: Response) => any,
110+
): Promise<any> => {
111+
if (!response.ok && onError) {
112+
return onError(response);
113+
}
114+
if (!response.body) return Promise.resolve(() => {});
115+
116+
const stream = response.body.getReader();
117+
const matcher = /\r?\n/;
118+
const decoder = new TextDecoder();
119+
let buffer = '';
120+
121+
const loop: () => Promise<undefined> = () =>
122+
stream.read().then(({ done, value }) => {
123+
if (done) {
124+
if (buffer.length > 0) processLine(JSON.parse(buffer));
125+
} else {
126+
const chunk = decoder.decode(value, {
127+
stream: true,
128+
});
129+
buffer += chunk;
130+
131+
const parts = buffer.split(matcher);
132+
buffer = parts.pop() ?? '';
133+
const validParts = parts.filter((p) => p);
134+
if (validParts.length !== 0) {
135+
for (const i of validParts) {
136+
const p = JSON.parse(i) as T;
137+
processLine(p);
138+
}
139+
return loop();
140+
}
141+
}
142+
});
143+
144+
return loop();
145+
};
146+
147+
export const getResult = async () => {
148+
const results: StreamEntry[] = [];
149+
150+
const streamResponse = await stream();
151+
if (streamResponse.status !== 200) return results;
152+
153+
// The promise is resolved when the stream is complete.
154+
await readStream(streamResponse.stream, (obj) => {
155+
// obj is typed as StreamEntry
156+
results.push(obj);
157+
});
158+
return results;
159+
};
160+
```

‎packages/core/src/writers/single-mode.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
} from '../utils';
1111
import { generateImportsForBuilder } from './generate-imports-for-builder';
1212
import { generateTarget } from './target';
13-
import { getOrvalGeneratedTypes } from './types';
13+
import { getOrvalGeneratedTypes, getTypedResponse } from './types';
1414

1515
export const writeSingleMode = async ({
1616
builder,
@@ -119,6 +119,11 @@ export const writeSingleMode = async ({
119119
data += '\n';
120120
}
121121

122+
if (implementation.includes('TypedResponse<')) {
123+
data += getTypedResponse();
124+
data += '\n';
125+
}
126+
122127
if (!output.schemas && needSchema) {
123128
data += generateModelsInline(builder.schemas);
124129
}

‎packages/core/src/writers/split-mode.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import {
99
upath,
1010
} from '../utils';
1111
import { generateTarget } from './target';
12-
import { getOrvalGeneratedTypes } from './types';
12+
import { getOrvalGeneratedTypes, getTypedResponse } from './types';
1313
import { getMockFileExtensionByTypeName } from '../utils/fileExtensions';
1414
import { generateImportsForBuilder } from './generate-imports-for-builder';
1515

@@ -137,6 +137,12 @@ export const writeSplitMode = async ({
137137

138138
if (implementation.includes('NonReadonly<')) {
139139
implementationData += getOrvalGeneratedTypes();
140+
implementationData += '\n';
141+
}
142+
143+
if (implementation.includes('TypedResponse<')) {
144+
implementationData += getTypedResponse();
145+
implementationData += '\n';
140146
}
141147

142148
implementationData += `\n${implementation}`;

0 commit comments

Comments
 (0)