From 1c0cd702d4eb07171c29444a11a36b7ff7360d69 Mon Sep 17 00:00:00 2001 From: Andrey Melikhov Date: Wed, 18 Dec 2024 13:05:57 +0300 Subject: [PATCH] Move from req (#1952) --- .../components/processor/data-fetcher.ts | 97 ++++++++++++++++--- src/server/components/charts-engine/types.ts | 26 ++++- .../url/distincts/build-distincts-body.ts | 28 +++--- .../middlewareAdapters/charts-with-dataset.ts | 1 + .../controls-with-dataset.ts | 9 +- .../request-with-dataset/request-dataset.ts | 63 +++++++----- 6 files changed, 165 insertions(+), 59 deletions(-) diff --git a/src/server/components/charts-engine/components/processor/data-fetcher.ts b/src/server/components/charts-engine/components/processor/data-fetcher.ts index 4a6438a0c9..4860918f2e 100644 --- a/src/server/components/charts-engine/components/processor/data-fetcher.ts +++ b/src/server/components/charts-engine/components/processor/data-fetcher.ts @@ -23,7 +23,7 @@ import { import {registry} from '../../../../registry'; import {config} from '../../constants'; import type {ChartsEngine} from '../../index'; -import type {Source, SourceConfig} from '../../types'; +import type {AdapterContext, Source, SourceConfig} from '../../types'; import {Request as RequestPromise} from '../request'; import {hideSensitiveData} from '../utils'; @@ -149,6 +149,29 @@ export type DataFetcherResult = { data?: any; }; +type ZitadelParams = { + accessToken?: string; + serviceUserAccessToken?: string; +}; + +function addZitadelHeaders({ + headers, + zitadelParams, +}: { + headers: IncomingHttpHeaders; + zitadelParams: ZitadelParams; +}) { + if (zitadelParams?.accessToken) { + Object.assign(headers, {authorization: `Bearer ${zitadelParams.accessToken}`}); + } + + if (zitadelParams?.serviceUserAccessToken) { + Object.assign(headers, { + [SERVICE_USER_ACCESS_TOKEN_HEADER]: zitadelParams.serviceUserAccessToken, + }); + } +} + export class DataFetcher { static fetch({ chartsEngine, @@ -186,6 +209,28 @@ export class DataFetcher { const queue = new PQueue({concurrency: CONCURRENT_REQUESTS_LIMIT}); const fetchPromisesList: (() => unknown)[] = []; + const isEmbed = req.headers[DL_EMBED_TOKEN_HEADER] !== undefined; + + const zitadelParams = ctx.config.isZitadelEnabled + ? { + accessToken: req.user?.accessToken, + serviceUserAccessToken: req.serviceUserAccessToken, + } + : undefined; + + const originalReqHeaders = { + xRealIP: req.headers['x-real-ip'], + xForwardedFor: req.headers['x-forwarded-for'], + xChartsFetcherVia: req.headers['x-charts-fetcher-via'], + referer: req.headers.referer, + }; + const adapterContext: AdapterContext = { + headers: { + ['x-forwarded-for']: req.headers['x-forwarded-for'], + cookie: req.headers.cookie, + }, + }; + Object.keys(sources).forEach((sourceName) => { const source = sources[sourceName]; @@ -205,6 +250,10 @@ export class DataFetcher { userLogin, iamToken, workbookId, + isEmbed, + zitadelParams, + originalReqHeaders, + adapterContext, }) : { sourceId: sourceName, @@ -420,6 +469,10 @@ export class DataFetcher { userLogin, iamToken, workbookId, + isEmbed, + zitadelParams, + originalReqHeaders, + adapterContext, }: { sourceName: string; source: Source; @@ -434,6 +487,15 @@ export class DataFetcher { userLogin?: string | null; iamToken?: string | null; workbookId?: WorkbookId; + isEmbed: boolean; + zitadelParams: ZitadelParams | undefined; + originalReqHeaders: { + xRealIP: IncomingHttpHeaders['x-real-ip']; + xForwardedFor: IncomingHttpHeaders['x-forwarded-for']; + xChartsFetcherVia: IncomingHttpHeaders['x-charts-fetcher-via']; + referer: IncomingHttpHeaders['referer']; + }; + adapterContext: AdapterContext; }) { const singleFetchingTimeout = chartsEngine.config.singleFetchingTimeout || DEFAULT_SINGLE_FETCHING_TIMEOUT; @@ -528,7 +590,7 @@ export class DataFetcher { const sourceConfig = DataFetcher.getSourceConfig({ chartsEngine, sourcePath: targetUri, - isEmbed: req.headers[DL_EMBED_TOKEN_HEADER] !== undefined, + isEmbed, }); if (!sourceConfig) { @@ -593,6 +655,15 @@ export class DataFetcher { }); } + if (sourceConfig.adapterWithContext) { + return sourceConfig.adapterWithContext({ + targetUri: croppedTargetUri, + sourceName, + adapterContext, + ctx, + }); + } + const headers: IncomingHttpHeaders = Object.assign( {}, { @@ -602,7 +673,7 @@ export class DataFetcher { ); if (sourceType === 'charts') { - const incomingHeader = req.headers['x-charts-fetcher-via'] || ''; + const incomingHeader = originalReqHeaders.xChartsFetcherVia || ''; const scriptName = req.body.params ? '/editor/' + req.body.params.name : req.body.path; @@ -627,8 +698,8 @@ export class DataFetcher { : scriptName; } - if (req.headers.referer) { - headers.referer = ctx.utils.redactSensitiveQueryParams(req.headers.referer); + if (originalReqHeaders.referer) { + headers.referer = ctx.utils.redactSensitiveQueryParams(originalReqHeaders.referer); } const proxyHeaders = ctx.config.chartsEngineConfig.dataFetcherProxiedHeaders || [ @@ -651,14 +722,8 @@ export class DataFetcher { headers[WORKBOOK_ID_HEADER] = workbookId; } - if (req.user?.accessToken) { - Object.assign(headers, {authorization: `Bearer ${req.user.accessToken}`}); - } - - if (req.serviceUserAccessToken) { - Object.assign(headers, { - [SERVICE_USER_ACCESS_TOKEN_HEADER]: req.serviceUserAccessToken, - }); + if (zitadelParams) { + addZitadelHeaders({headers, zitadelParams}); } if (passedCredentials) { @@ -678,7 +743,7 @@ export class DataFetcher { if (extraHeaders) { if (typeof extraHeaders === 'function') { - const extraHeadersResult = extraHeaders(req); + const extraHeadersResult = extraHeaders(); Object.assign(headers, extraHeadersResult); } else if (typeof extraHeaders === 'object') { @@ -726,7 +791,7 @@ export class DataFetcher { } if (ctx.config.appEnv !== 'development') { - requestOptions.headers['x-forwarded-for'] = req.headers['x-forwarded-for']; + requestOptions.headers['x-forwarded-for'] = originalReqHeaders.xForwardedFor; } if (isSourceWithMiddlewareUrl(source)) { @@ -767,7 +832,7 @@ export class DataFetcher { const publicSourceData = hideSensitiveData(sourceData); if (!requestOptions.headers['x-real-ip']) { - requestOptions.headers['x-real-ip'] = req.headers['x-real-ip']; + requestOptions.headers['x-real-ip'] = originalReqHeaders.xRealIP; } const traceId = ctx.getTraceId(); diff --git a/src/server/components/charts-engine/types.ts b/src/server/components/charts-engine/types.ts index 1bbc15c11c..f472f11eaf 100644 --- a/src/server/components/charts-engine/types.ts +++ b/src/server/components/charts-engine/types.ts @@ -1,4 +1,4 @@ -import type {OutgoingHttpHeaders} from 'http'; +import type {IncomingHttpHeaders, OutgoingHttpHeaders} from 'http'; import type {AppMiddleware, AppRouteDescription, Request, Response} from '@gravity-ui/expresskit'; import type {HttpMethod} from '@gravity-ui/expresskit/dist/types'; @@ -125,6 +125,13 @@ export type Source> = { sourceArgs?: SourcesArgs; }; +export type AdapterContext = { + headers: { + cookie: IncomingHttpHeaders['cookie']; + ['x-forwarded-for']: IncomingHttpHeaders['x-forwarded-for']; + }; +}; + export type SourceConfig = { description?: { title: { @@ -139,12 +146,15 @@ export type SourceConfig = { uiEndpointFormatter?: (url: string, sourceData?: Source['data']) => string | null; uiEndpoint?: string; passedCredentials?: Record; - extraHeaders?: Record | ((req: Request) => Record); + extraHeaders?: Record | (() => Record); sourceType?: string; dataEndpoint?: string; preprocess?: (url: string) => string; allowedMethods?: ('GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE')[]; + /** + * @deprecated + **/ adapter?: ({ targetUri, sourceName, @@ -157,6 +167,18 @@ export type SourceConfig = { ctx: AppContext; }) => unknown; + adapterWithContext?: ({ + targetUri, + sourceName, + adapterContext, + ctx, + }: { + targetUri: string; + sourceName: string; + adapterContext: AdapterContext; + ctx: AppContext; + }) => unknown; + middlewareAdapter?: (args: MiddlewareSourceAdapterArgs) => Promise; check?: (targetUri: string, params?: Request['body']['params']) => Promise; diff --git a/src/server/modes/charts/plugins/control/url/distincts/build-distincts-body.ts b/src/server/modes/charts/plugins/control/url/distincts/build-distincts-body.ts index a6b817e182..6099936921 100644 --- a/src/server/modes/charts/plugins/control/url/distincts/build-distincts-body.ts +++ b/src/server/modes/charts/plugins/control/url/distincts/build-distincts-body.ts @@ -1,4 +1,4 @@ -import type {Request} from '@gravity-ui/expresskit'; +import type {AppContext} from '@gravity-ui/nodekit'; import type { ApiV2Filter, @@ -85,14 +85,14 @@ export const getDistinctsRequestBody = ({ shared, params, datasetFields, - req, + ctx, }: { shared: ControlShared; params: StringParams; datasetFields: PartialDatasetField[]; - req?: Request; + ctx?: AppContext; }): ApiV2RequestBody => { - req?.ctx?.log?.('CONTROLS_START_PREPARING_DISTINCTS_BODY'); + ctx?.log('CONTROLS_START_PREPARING_DISTINCTS_BODY'); const targetParam = shared.param; const where: { @@ -101,7 +101,7 @@ export const getDistinctsRequestBody = ({ values: string[]; }[] = []; - req?.ctx?.log?.('CONTROLS_START_MAPPING_DATASET_FIELDS'); + ctx?.log('CONTROLS_START_MAPPING_DATASET_FIELDS'); const datasetFieldsMap = datasetFields.reduce( (acc, field) => { @@ -117,29 +117,29 @@ export const getDistinctsRequestBody = ({ {} as Record, ); - req?.ctx?.log?.('CONTROLS_END_MAPPING_DATASET_FIELDS'); + ctx?.log('CONTROLS_END_MAPPING_DATASET_FIELDS'); - req?.ctx?.log?.('CONTROLS_START_TRANSFORMING_PARAMS'); + ctx?.log?.('CONTROLS_START_TRANSFORMING_PARAMS'); const urlSearchParams = transformParamsToUrlParams(params); - req?.ctx?.log?.('CONTROLS_END_TRANSFORMING_PARAMS'); + ctx?.log?.('CONTROLS_END_TRANSFORMING_PARAMS'); - req?.ctx?.log?.('CONTROLS_START_SPLIT_PARAMS'); + ctx?.log?.('CONTROLS_START_SPLIT_PARAMS'); const {filtersParams, parametersParams} = splitParamsToParametersAndFilters( urlSearchParams, datasetFields, ); - req?.ctx?.log?.('CONTROLS_START_TRANSFORMING_PARAMS'); + ctx?.log?.('CONTROLS_START_TRANSFORMING_PARAMS'); const transformedFilterParams = transformUrlParamsToParams(filtersParams); const transformedParametersParams = transformUrlParamsToParams(parametersParams); - req?.ctx?.log?.('CONTROLS_END_TRANSFORMING_PARAMS'); + ctx?.log?.('CONTROLS_END_TRANSFORMING_PARAMS'); - req?.ctx?.log('CONTROLS_START_PROCESSING_FILTERS'); + ctx?.log('CONTROLS_START_PROCESSING_FILTERS'); Object.keys(transformedFilterParams).forEach((param) => { if (param === targetParam) { @@ -190,7 +190,7 @@ export const getDistinctsRequestBody = ({ } }); - req?.ctx?.log('CONTROLS_END_PROCESSING_FILTERS'); + ctx?.log('CONTROLS_END_PROCESSING_FILTERS'); const apiV2RequestBody = buildDistinctsBodyRequest({ where, @@ -199,7 +199,7 @@ export const getDistinctsRequestBody = ({ datasetFieldsMap, }); - req?.ctx?.log('CONTROLS_END_PREPARING_DISTINCTS_BODY'); + ctx?.log('CONTROLS_END_PREPARING_DISTINCTS_BODY'); return apiV2RequestBody; }; diff --git a/src/server/modes/charts/plugins/request-with-dataset/middlewareAdapters/charts-with-dataset.ts b/src/server/modes/charts/plugins/request-with-dataset/middlewareAdapters/charts-with-dataset.ts index 166256e9e2..2eb899c3b3 100644 --- a/src/server/modes/charts/plugins/request-with-dataset/middlewareAdapters/charts-with-dataset.ts +++ b/src/server/modes/charts/plugins/request-with-dataset/middlewareAdapters/charts-with-dataset.ts @@ -47,6 +47,7 @@ export default async ( datasetId, workbookId: workbookId ?? null, req, + ctx: req.ctx, cacheClient, userId, iamToken, diff --git a/src/server/modes/charts/plugins/request-with-dataset/middlewareAdapters/controls-with-dataset.ts b/src/server/modes/charts/plugins/request-with-dataset/middlewareAdapters/controls-with-dataset.ts index 8eeacb7a38..bbe095bf85 100644 --- a/src/server/modes/charts/plugins/request-with-dataset/middlewareAdapters/controls-with-dataset.ts +++ b/src/server/modes/charts/plugins/request-with-dataset/middlewareAdapters/controls-with-dataset.ts @@ -21,6 +21,8 @@ export default async ( pluginOptions, } = args; + const ctx = req.ctx; + const cacheClient = ChartsEngine.cacheClient as Cache; const datasetId = source.datasetId || ''; @@ -29,6 +31,7 @@ export default async ( datasetId, workbookId: workbookId ?? null, req, + ctx, cacheClient, userId, iamToken, @@ -38,7 +41,7 @@ export default async ( const datasetFields = datasetFieldsResponse.datasetFields; - req.ctx.log('CONTROLS_DATASET_FIELDS_RECEIVED', { + ctx.log('CONTROLS_DATASET_FIELDS_RECEIVED', { count: datasetFields.length, }); @@ -46,10 +49,10 @@ export default async ( params: source.sourceArgs.params, shared: source.sourceArgs.shared as unknown as ControlShared, datasetFields, - req, + ctx, }); - req.ctx.log('CONTROLS_DATASET_FIELDS_PROCESSED'); + ctx.log('CONTROLS_DATASET_FIELDS_PROCESSED'); return { ...source, diff --git a/src/server/modes/charts/plugins/request-with-dataset/request-dataset.ts b/src/server/modes/charts/plugins/request-with-dataset/request-dataset.ts index 2325089782..889316fd6c 100644 --- a/src/server/modes/charts/plugins/request-with-dataset/request-dataset.ts +++ b/src/server/modes/charts/plugins/request-with-dataset/request-dataset.ts @@ -1,4 +1,6 @@ import type {Request} from '@gravity-ui/expresskit'; +import type {AppContext} from '@gravity-ui/nodekit'; +import {REQUEST_ID_PARAM_NAME} from '@gravity-ui/nodekit'; import {isObject} from 'lodash'; import isNumber from 'lodash/isNumber'; @@ -17,14 +19,23 @@ export const DEFAULT_CACHE_TTL = 30; const getStatusFromError = (error: unknown) => typeof error === 'object' && error !== null && 'status' in error && error.status; -export const getDatasetFieldsById = async ( - datasetId: string, - workbookId: string | null, - req: Request, - rejectFetchingSource: (reason?: any) => void, - iamToken?: string, - pluginOptions?: ConfigurableRequestWithDatasetPluginOptions, -): Promise => { +const getDatasetFieldsById = async ({ + datasetId, + workbookId, + req, + ctx, + rejectFetchingSource, + iamToken, + pluginOptions, +}: { + datasetId: string; + workbookId: string | null; + req: Request; + ctx: AppContext; + rejectFetchingSource: (reason?: any) => void; + iamToken?: string; + pluginOptions?: ConfigurableRequestWithDatasetPluginOptions; +}): Promise => { const {gatewayApi} = registry.getGatewayApi(); const requestDatasetFields = @@ -36,17 +47,17 @@ export const getDatasetFieldsById = async ( const response = req.headers[DL_EMBED_TOKEN_HEADER] ? await requestDatasetFieldsByToken({ - ctx: req.ctx, + ctx, headers, - requestId: req.id, + requestId: ctx.get(REQUEST_ID_PARAM_NAME) || '', args: { dataSetId: datasetId, }, }) : await requestDatasetFields({ - ctx: req.ctx, + ctx: ctx, headers, - requestId: req.id, + requestId: ctx.get(REQUEST_ID_PARAM_NAME) || '', authArgs: {iamToken}, args: { dataSetId: datasetId, @@ -63,7 +74,7 @@ export const getDatasetFieldsById = async ( const message = error.message as string; preparedError = new Error(message); } - req.ctx.logError('FAILED_TO_RECEIVE_FIELDS', preparedError); + ctx.logError('FAILED_TO_RECEIVE_FIELDS', preparedError); const status = getStatusFromError(error); if (isNumber(status) && status < 500) { rejectFetchingSource({ @@ -79,6 +90,7 @@ export const getDatasetFields = async (args: { datasetId: string; workbookId: WorkbookId; req: Request; + ctx: AppContext; iamToken?: string; cacheClient: Cache; userId: string | null; @@ -90,6 +102,7 @@ export const getDatasetFields = async (args: { workbookId, cacheClient, req, + ctx, userId, iamToken, rejectFetchingSource, @@ -98,7 +111,7 @@ export const getDatasetFields = async (args: { const cacheKey = `${datasetId}__${userId}`; - req.ctx.log('DATASET_FOR_CHARTS_MIDDLEWARE', {cacheKey}); + ctx.log('DATASET_FOR_CHARTS_MIDDLEWARE', {cacheKey}); let datasetFields: PartialDatasetField[]; let revisionId: string; @@ -109,18 +122,19 @@ export const getDatasetFields = async (args: { if (cacheResponse.status === Cache.OK) { datasetFields = cacheResponse.data.datasetFields; revisionId = cacheResponse.data.revisionId; - req.ctx.log('DATASET_FIELDS_WAS_RECEIVED_FROM_CACHE'); + ctx.log('DATASET_FIELDS_WAS_RECEIVED_FROM_CACHE'); } else { - req.ctx.log('DATASET_FIELDS_IN_CACHE_WAS_NOT_FOUND'); + ctx.log('DATASET_FIELDS_IN_CACHE_WAS_NOT_FOUND'); - const response = await getDatasetFieldsById( + const response = await getDatasetFieldsById({ datasetId, workbookId, req, + ctx, rejectFetchingSource, iamToken, pluginOptions, - ); + }); datasetFields = response.fields; revisionId = response.revision_id; cacheClient @@ -131,32 +145,33 @@ export const getDatasetFields = async (args: { }) .then((setCacheResponse) => { if (setCacheResponse.status === Cache.OK) { - req.ctx.log('SET_DATASET_IN_CACHE_SUCCESS'); + ctx.log('SET_DATASET_IN_CACHE_SUCCESS'); } else { - req.ctx.logError( + ctx.logError( 'SET_DATASET_FIELDS_IN_CACHE_FAILED', new Error(setCacheResponse.message), ); } }) .catch((error) => { - req.ctx.logError('SET_DATASET_FIELDS_UNHANDLED_ERROR', error); + ctx.logError('SET_DATASET_FIELDS_UNHANDLED_ERROR', error); }); } } else { - const response = await getDatasetFieldsById( + const response = await getDatasetFieldsById({ datasetId, workbookId, req, + ctx, rejectFetchingSource, iamToken, pluginOptions, - ); + }); datasetFields = response.fields; revisionId = response.revision_id; } - req.ctx.log('DATASET_FIELDS_WAS_SUCCESSFULLY_PROCESSED'); + ctx.log('DATASET_FIELDS_WAS_SUCCESSFULLY_PROCESSED'); return {datasetFields, revisionId}; };