diff --git a/CHANGELOG.md b/CHANGELOG.md index 429f274b7..cd0153581 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.2.0] - 2023-10-02: V3 integration improvements; bug fixes + +### Added + +- WP-211: App Form updates to allow target path (#857) +- WP-272: Include username in Onboarding Admin user listing (#861) + +### Changed + +- WP-189 Handle timeout exit code for interactive app jobs (#851) +- WP-163 Compress Archive Path Fix (#846) +- WP-105: create common utils function (#850) +- WP-172: Minimize unit test warnings (#855) +- WP-62: Changed upload function to use TAPIS file insert api (#859) + +### Fixed +- WP-249 Shared Workspace Copy Bug Fix (#858) +- WP-262 Workspace file operations bug fixes (#862) +- WP-276: Fixed Data Files Add button dropdown off-centered UI (#863) +- Quick: handle missing default system; enable work as default system locally (#867) +- WP-52 Jobs View Infinite Scroll Fix (#865) +- WP-228: Fixed sorting for system list (#860) +- WP-209: fix deprecated warnings (part II) (#852) + + ## [3.1.2] - 2023-08-22: Secure user search endpoint ### Fixed @@ -929,6 +954,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 v1.0.0 Production release as of Feb 28, 2020. [unreleased]: https://github.com/TACC/Core-Portal/compare/v3.1.2...HEAD +[3.2.0]: https://github.com/TACC/Core-Portal/releases/tag/v3.2.0 [3.1.2]: https://github.com/TACC/Core-Portal/releases/tag/v3.1.2 [3.1.1]: https://github.com/TACC/Core-Portal/releases/tag/v3.1.1 [3.1.0]: https://github.com/TACC/Core-Portal/releases/tag/v3.1.0 diff --git a/client/src/components/Applications/AppForm/AppForm.jsx b/client/src/components/Applications/AppForm/AppForm.jsx index 4cb3cfbf9..b2b7ed793 100644 --- a/client/src/components/Applications/AppForm/AppForm.jsx +++ b/client/src/components/Applications/AppForm/AppForm.jsx @@ -19,10 +19,14 @@ import { Link } from 'react-router-dom'; import { getSystemName } from 'utils/systems'; import FormSchema from './AppFormSchema'; import { + checkAndSetDefaultTargetPath, + isTargetPathField, + getInputFieldFromTargetPathField, getQueueMaxMinutes, getMaxMinutesValidation, getNodeCountValidation, getCoresPerNodeValidation, + getTargetPathFieldName, updateValuesForQueue, } from './AppFormUtils'; import DataFilesSelectModal from '../../DataFiles/DataFilesModals/DataFilesSelectModal'; @@ -201,7 +205,7 @@ export const AppSchemaForm = ({ app }) => { const hasCorral = configuration.length && ['corral.tacc.utexas.edu', 'data.tacc.utexas.edu'].some((s) => - defaultHost.endsWith(s) + defaultHost?.endsWith(s) ); return { allocations: matchingExecutionHost @@ -459,17 +463,49 @@ export const AppSchemaForm = ({ app }) => { onSubmit={(values, { setSubmitting, resetForm }) => { const job = cloneDeep(values); - job.fileInputs = Object.entries(job.fileInputs) - .map(([k, v]) => { - // filter out read only inputs. 'FIXED' inputs are tracked as readOnly - if ( - Object.hasOwn(appFields.fileInputs, k) && - appFields.fileInputs[k].readOnly - ) - return; - return { name: k, sourceUrl: v }; - }) - .filter((fileInput) => fileInput && fileInput.sourceUrl); // filter out any empty values + // Transform input field values into format that jobs service wants. + // File Input structure will have 2 fields if target path is required by the app. + // field 1 - has source url + // field 2 - has target path for the source url. + // tapis wants only 1 field with 2 properties - source url and target path. + // The logic below handles that scenario by merging the related fields into 1 field. + job.fileInputs = Object.values( + Object.entries(job.fileInputs) + .map(([k, v]) => { + // filter out read only inputs. 'FIXED' inputs are tracked as readOnly + if ( + Object.hasOwn(appFields.fileInputs, k) && + appFields.fileInputs[k].readOnly + ) + return; + return { + name: k, + sourceUrl: !isTargetPathField(k) ? v : null, + targetDir: isTargetPathField(k) ? v : null, + }; + }) + .reduce((acc, entry) => { + // merge input field and targetPath fields into one. + const key = getInputFieldFromTargetPathField(entry.name); + if (!acc[key]) { + acc[key] = {}; + } + acc[key]['name'] = key; + acc[key]['sourceUrl'] = + acc[key]['sourceUrl'] ?? entry.sourceUrl; + acc[key]['targetPath'] = + acc[key]['targetPath'] ?? entry.targetDir; + return acc; + }, {}) + ) + .flat() + .filter((fileInput) => fileInput.sourceUrl) // filter out any empty values + .map((fileInput) => { + fileInput.targetPath = checkAndSetDefaultTargetPath( + fileInput.targetPath + ); + return fileInput; + }); job.parameterSet = Object.assign( {}, @@ -559,8 +595,15 @@ export const AppSchemaForm = ({ app }) => { {Object.entries(appFields.fileInputs).map( ([name, field]) => { - // TODOv3 handle fileInputArrays https://jira.tacc.utexas.edu/browse/TV3-81 - return ( + // TODOv3 handle fileInputArrays https://jira.tacc.utexas.edu/browse/WP-81 + return isTargetPathField(name) ? ( + + ) : ( { const appFields = { @@ -97,6 +101,9 @@ const FormSchema = (app) => { } ); + // The default is to not show target path for file inputs. + const showTargetPathForFileInputs = + app.definition.notes.showTargetPath ?? false; (app.definition.jobAttributes.fileInputs || []).forEach((i) => { const input = i; /* TODOv3 consider hidden file inputs https://jira.tacc.utexas.edu/browse/WP-102 @@ -131,6 +138,33 @@ const FormSchema = (app) => { input.sourceUrl === null || typeof input.sourceUrl === 'undefined' ? '' : input.sourceUrl; + + // Add targetDir for all sourceUrl + if (!showTargetPathForFileInputs) { + return; + } + const targetPathName = getTargetPathFieldName(input.name); + appFields.schema.fileInputs[targetPathName] = Yup.string(); + appFields.schema.fileInputs[targetPathName] = appFields.schema.fileInputs[ + targetPathName + ].matches( + /^tapis:\/\//g, + "Input file Target Directory must be a valid Tapis URI, starting with 'tapis://'" + ); + + appFields.schema.fileInputs[targetPathName] = false; + appFields.fileInputs[targetPathName] = { + label: 'Target Path for ' + input.name, + description: + 'The name of the ' + + input.name + + ' after it is copied to the target system, but before the job is run. Leave this value blank to just use the name of the input file.', + required: false, + readOnly: field.readOnly, + type: 'text', + }; + appFields.defaults.fileInputs[targetPathName] = + checkAndSetDefaultTargetPath(input.targetPath); }); return appFields; }; diff --git a/client/src/components/Applications/AppForm/AppFormUtils.js b/client/src/components/Applications/AppForm/AppFormUtils.js index 812fe3379..1b5b30c7e 100644 --- a/client/src/components/Applications/AppForm/AppFormUtils.js +++ b/client/src/components/Applications/AppForm/AppFormUtils.js @@ -1,6 +1,8 @@ import * as Yup from 'yup'; import { getSystemName } from 'utils/systems'; +export const TARGET_PATH_FIELD_PREFIX = '_TargetPath_'; + export const getQueueMaxMinutes = (app, queueName) => { return app.exec_sys.batchLogicalQueues.find((q) => q.name === queueName) .maxMinutes; @@ -165,3 +167,57 @@ export const updateValuesForQueue = (app, values) => { return updatedValues; }; + +/** + * Get the field name used for target path in AppForm + * + * @function + * @param {String} inputFieldName + * @returns {String} field Name prefixed with target path + */ +export const getTargetPathFieldName = (inputFieldName) => { + return TARGET_PATH_FIELD_PREFIX + inputFieldName; +}; + +/** + * Whether a field name is a system defined field for Target Path + * + * @function + * @param {String} inputFieldName + * @returns {String} field Name suffixed with target path + */ +export const isTargetPathField = (inputFieldName) => { + return inputFieldName && inputFieldName.startsWith(TARGET_PATH_FIELD_PREFIX); +}; + +/** + * From target path field name, derive the original input field name. + * + * @function + * @param {String} targetPathFieldName + * @returns {String} actual field name + */ +export const getInputFieldFromTargetPathField = (targetPathFieldName) => { + return targetPathFieldName.replace(TARGET_PATH_FIELD_PREFIX, ''); +}; + +/** + * Sets the default value if target path is not set. + * + * @function + * @param {String} targetPathFieldValue + * @returns {String} target path value + */ +export const checkAndSetDefaultTargetPath = (targetPathFieldValue) => { + if (targetPathFieldValue === null || targetPathFieldValue === undefined) { + return '*'; + } + + targetPathFieldValue = targetPathFieldValue.trim(); + + if (targetPathFieldValue.trim() === '') { + return '*'; + } + + return targetPathFieldValue; +}; diff --git a/client/src/components/DataFiles/DataFilesModals/DataFilesMoveModal.jsx b/client/src/components/DataFiles/DataFilesModals/DataFilesMoveModal.jsx index 0f079795a..1faa2995e 100644 --- a/client/src/components/DataFiles/DataFilesModals/DataFilesMoveModal.jsx +++ b/client/src/components/DataFiles/DataFilesModals/DataFilesMoveModal.jsx @@ -50,10 +50,7 @@ const DataFilesMoveModal = React.memo(() => { const onOpened = () => { fetchListing({ - api: 'tapis', - scheme: 'private', - system: selectedSystem.system, - path: `${selectedSystem.homeDir || ''}`, + ...params, }); }; @@ -68,7 +65,7 @@ const DataFilesMoveModal = React.memo(() => { setDisabled(true); move({ destSystem: system, - destPath: path, + destPath: path || '/', callback: reloadPage, }); }, diff --git a/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.scss b/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.scss index b343e4381..9f4fac877 100644 --- a/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.scss +++ b/client/src/components/DataFiles/DataFilesSidebar/DataFilesSidebar.scss @@ -12,7 +12,7 @@ padding-left: var(--horizontal-buffer); } #data-files-add { - width: 140px; + width: 174px; } .data-files-nav { @@ -36,6 +36,7 @@ border-right: 10px solid transparent; border-bottom: 10px solid var(--global-color-accent--normal); border-left: 10px solid transparent; + margin-left: 20px; content: ''; } .dropdown-menu::after { @@ -45,6 +46,7 @@ border-right: 9px solid transparent; border-bottom: 9px solid #ffffff; border-left: 9px solid transparent; + margin-left: 20px; content: ''; } diff --git a/client/src/components/Jobs/Jobs.jsx b/client/src/components/Jobs/Jobs.jsx index bbef04a96..1945d1bf7 100644 --- a/client/src/components/Jobs/Jobs.jsx +++ b/client/src/components/Jobs/Jobs.jsx @@ -75,11 +75,14 @@ function JobsView({ const infiniteScrollCallback = useCallback(() => { // TODOv3: dropV2Jobs const dispatchType = version === 'v3' ? 'GET_JOBS' : 'GET_V2_JOBS'; - dispatch({ - type: dispatchType, - params: { offset: jobs.length, queryString: query.query_string || '' }, - }); - }, [dispatch, jobs, query.query_string]); + + if (!isJobLoading) { + dispatch({ + type: dispatchType, + params: { offset: jobs.length, queryString: query.query_string || '' }, + }); + } + }, [dispatch, jobs, query.query_string, isJobLoading]); const jobDetailLink = useCallback( ({ @@ -217,22 +220,20 @@ function JobsView({ disabled={isJobLoading || isNotificationLoading} /> )} -
- - {noDataText} - - } - getRowProps={rowProps} - columnMemoProps={[version]} /* TODOv3: dropV2Jobs. */ - /> -
+ + {noDataText} + + } + getRowProps={rowProps} + columnMemoProps={[version]} /* TODOv3: dropV2Jobs. */ + /> ); } diff --git a/client/src/components/Jobs/Jobs.test.js b/client/src/components/Jobs/Jobs.test.js index ea30d1934..05dcb53f6 100644 --- a/client/src/components/Jobs/Jobs.test.js +++ b/client/src/components/Jobs/Jobs.test.js @@ -1,5 +1,5 @@ import React from 'react'; -import { render } from '@testing-library/react'; +import { fireEvent, render, waitFor } from '@testing-library/react'; import Jobs from './Jobs'; import { createMemoryHistory } from 'history'; import { default as jobsList } from './Jobs.fixture'; @@ -135,4 +135,35 @@ describe('Jobs View', () => { const { getByText } = renderJobsComponent(store, history); expect(getByText(/unable to retrieve your jobs/)).toBeDefined(); }); + + it('should dispatch another get jobs event on scroll with proper offset', async () => { + const store = mockStore({ + notifications, + jobs: { ...jobs, list: jobsList }, + workbench: { ...workbench, config: { hideDataFiles: false } }, + apps: { + appIcons: {}, + }, + }); + + const { container } = render( + + + + + + ); + + const scrollContainer = container.querySelector('.table-container'); + + fireEvent.scroll(scrollContainer, { target: { scrollTop: 1 } }); + + expect(store.getActions()).toEqual([ + { type: 'GET_JOBS', params: { offset: 0, queryString: '' } }, + { + type: 'GET_JOBS', + params: { offset: jobsList.length, queryString: '' }, + }, + ]); + }); }); diff --git a/client/src/components/_common/InfiniteScrollTable/InfiniteScrollTable.jsx b/client/src/components/_common/InfiniteScrollTable/InfiniteScrollTable.jsx index f8496eeb8..a3bbe85e5 100644 --- a/client/src/components/_common/InfiniteScrollTable/InfiniteScrollTable.jsx +++ b/client/src/components/_common/InfiniteScrollTable/InfiniteScrollTable.jsx @@ -62,51 +62,60 @@ const InfiniteScrollTable = ({ useTable({ columns, data }); const onScroll = ({ target }) => { - const bottom = - target.scrollHeight - target.scrollTop === target.clientHeight; + const scrollbarHeight = target.offsetHeight - target.clientHeight; + const clientRectHeight = target.getBoundingClientRect().height; + const clientCalcHeight = clientRectHeight - scrollbarHeight; + const difference = Math.floor(target.scrollHeight - target.scrollTop); + + const bottom = difference <= clientCalcHeight; + if (bottom && target.scrollTop > 0) { onInfiniteScroll(tableData.length); } }; return ( - - - {headerGroups.map((headerGroup) => ( - - {headerGroup.headers.map((column) => ( - - ))} - - ))} - - - {rows.map((row) => { - prepareRow(row); - return ( - - {row.cells.map((cell) => { - return ( - - ); - })} +
+
{column.render('Header')}
- {cell.render('Cell')} -
+ + {headerGroups.map((headerGroup) => ( + + {headerGroup.headers.map((column) => ( + + ))} - ); - })} - - - -
{column.render('Header')}
+ ))} + + + {rows.map((row) => { + prepareRow(row); + return ( + + {row.cells.map((cell) => { + return ( + + {cell.render('Cell')} + + ); + })} + + ); + })} + + + + + ); }; diff --git a/client/src/components/_common/InfiniteScrollTable/InfiniteScrollTable.scss b/client/src/components/_common/InfiniteScrollTable/InfiniteScrollTable.scss index 55b98d918..4361f7586 100644 --- a/client/src/components/_common/InfiniteScrollTable/InfiniteScrollTable.scss +++ b/client/src/components/_common/InfiniteScrollTable/InfiniteScrollTable.scss @@ -79,3 +79,8 @@ width: 100%; } } + +.table-container { + height: 100%; + overflow: scroll; +} diff --git a/client/src/redux/sagas/datafiles.sagas.js b/client/src/redux/sagas/datafiles.sagas.js index 730a63c44..2ef66a565 100644 --- a/client/src/redux/sagas/datafiles.sagas.js +++ b/client/src/redux/sagas/datafiles.sagas.js @@ -254,7 +254,7 @@ export function* renameFile(action) { action.payload.api, action.payload.scheme, file.system, - file.path, + '/' + file.path, action.payload.newName ); yield put({ diff --git a/server/portal/apps/datafiles/views.py b/server/portal/apps/datafiles/views.py index e85d42069..8f0e769cd 100644 --- a/server/portal/apps/datafiles/views.py +++ b/server/portal/apps/datafiles/views.py @@ -48,7 +48,7 @@ def get(self, request): if 'homeDir' in system else system for system in portal_systems ] - default_system = settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM + default_system = settings.PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM or settings.PORTAL_DATAFILES_STORAGE_SYSTEMS[0] if default_system: system_id = default_system.get('system') system_def = request.user.tapis_oauth.client.systems.getSystem(systemId=system_id, select='host') diff --git a/server/portal/apps/workspace/api/views.py b/server/portal/apps/workspace/api/views.py index 9a94db829..f76b36402 100644 --- a/server/portal/apps/workspace/api/views.py +++ b/server/portal/apps/workspace/api/views.py @@ -179,7 +179,7 @@ def listing(self, client, request): data = client.jobs.getJobSearchList( limit=limit, - startAfter=offset, + skip=offset, orderBy='lastUpdated(desc),name(asc)', _tapis_query_parameters={'tags.contains': f'portalName: {portal_name}'}, select='allAttributes' diff --git a/server/portal/settings/settings.py b/server/portal/settings/settings.py index e2e82770f..d4c6ae458 100644 --- a/server/portal/settings/settings.py +++ b/server/portal/settings/settings.py @@ -512,7 +512,7 @@ PORTAL_DATAFILES_STORAGE_SYSTEMS = getattr( settings_custom, '_PORTAL_DATAFILES_STORAGE_SYSTEMS', [] ) -PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM = next((sys for sys in PORTAL_DATAFILES_STORAGE_SYSTEMS if sys['default'] is True), None) +PORTAL_DATAFILES_DEFAULT_STORAGE_SYSTEM = next((sys for sys in PORTAL_DATAFILES_STORAGE_SYSTEMS if sys.get('default')), None) PORTAL_SEARCH_MANAGERS = { 'my-data': 'portal.apps.search.api.managers.private_data_search.PrivateDataSearchManager', diff --git a/server/portal/settings/settings_default.py b/server/portal/settings/settings_default.py index 433b52d3b..443520033 100644 --- a/server/portal/settings/settings_default.py +++ b/server/portal/settings/settings_default.py @@ -59,22 +59,21 @@ _PORTAL_DATAFILES_STORAGE_SYSTEMS = [ { - 'name': 'My Data (Corral)', - 'system': 'cloud.data', + 'name': 'My Data (Work)', + 'system': 'frontera', 'scheme': 'private', 'api': 'tapis', - 'homeDir': '/home/{username}', + 'homeDir': '/work/{tasdir}', 'icon': None, - 'keyservice': True, 'default': True }, { - 'name': 'My Data (Work)', + 'name': 'My Data (Scratch)', 'system': 'frontera', 'scheme': 'private', 'api': 'tapis', - 'homeDir': '/work/{tasdir}', - 'icon': None, + 'homeDir': '/scratch1/{tasdir}', + 'icon': None }, { 'name': 'My Data (Frontera)',