diff --git a/CHANGELOG.md b/CHANGELOG.md index fdd1e06..12c5b7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,110 +1,131 @@ - # Change Log for Visual Studio Code - Offline Gallery and Updater +## [1.0.24] - 2023-06-05 + +### Fixed + +- Improvements to requests session handling to prevent ConnectionErrors due to repeated connections. Thanks @tomer953 for reporting. + +### Added + +- Note about Firefox in Readme.md. Thanks @jmorcate for highlighting this gap. + +### Changed + +- Sort gallery listings with simple python sort. +- Removed deprecated logzero dependency, switched to logging. Thanks @bdsoha for the implementation and note. + ## [1.0.23] - 2022-11-09 ### Fixed - - @forky2 resolved an issue related to incorrect version ordering (from reverse-alphanumberical to reverse-chronological), which prevented extensions updating correctly by vscode clients. +- @forky2 resolved an issue related to incorrect version ordering (from reverse-alphanumberical to reverse-chronological), which prevented extensions updating correctly by vscode clients. ## [1.0.22] - 2022-10-31 ### Added - - @maxtruxa added support for specifying docker container environment variable `SSLARGS` to control SSL arguments, or disable SSL by setting `BIND=0.0.0.0:80` and `SSLARGS=` (empty). + +- @maxtruxa added support for specifying docker container environment variable `SSLARGS` to control SSL arguments, or disable SSL by setting `BIND=0.0.0.0:80` and `SSLARGS=` (empty). ### Changed - - @Precioussheep improved consistency of the codebase, reducing bonus code and added typing. +- @Precioussheep improved consistency of the codebase, reducing bonus code and added typing. ## [1.0.21] - 2022-08-08 ### Added - - @tomer953 added support for fetching a specified number of recommended extensions `--total-recommended`. - - @Ebsan added support for fetching pre-release extensions `--prerelease-extensions` and fix fetching other extensions [#31](https://github.com/LOLINTERNETZ/vscodeoffline/issues/31). - - @Ebsan added support for specifying which Visual Studio Code version to masquerade as when fetching extensions `--vscode-version`. + +- @tomer953 added support for fetching a specified number of recommended extensions `--total-recommended`. +- @Ebsan added support for fetching pre-release extensions `--prerelease-extensions` and fix fetching other extensions [#31](https://github.com/LOLINTERNETZ/vscodeoffline/issues/31). +- @Ebsan added support for specifying which Visual Studio Code version to masquerade as when fetching extensions `--vscode-version`. ### Changed - - Merge dependabot suggestions for CI pipeline updates. - - Utilise individual requests, rather than a Requests session, for fetching extensions to improve stability of fetch process. Should resolve [#33](https://github.com/LOLINTERNETZ/vscodeoffline/issues/33). Thanks @Ebsan for the fix and @annieherram for reporting. - - Updated build-in certificate and key to update its expiry [#37](https://github.com/LOLINTERNETZ/vscodeoffline/issues/37). Included CA chain aswell. Thanks for reporting @Ebsan. - - Removed platform suport for ia32 builds, as they're no longer provided since ~1.35. - - Split out this changelog. + +- Merge dependabot suggestions for CI pipeline updates. +- Utilise individual requests, rather than a Requests session, for fetching extensions to improve stability of fetch process. Should resolve [#33](https://github.com/LOLINTERNETZ/vscodeoffline/issues/33). Thanks @Ebsan for the fix and @annieherram for reporting. +- Updated build-in certificate and key to update its expiry [#37](https://github.com/LOLINTERNETZ/vscodeoffline/issues/37). Included CA chain aswell. Thanks for reporting @Ebsan. +- Removed platform suport for ia32 builds, as they're no longer provided since ~1.35. +- Split out this changelog. ### Fixed - - @tomer953 removed a duplicate flag to QueryFlags. - - @Ebsan fixed an issue with downloading cross-platform extensions [#24](https://github.com/LOLINTERNETZ/vscodeoffline/issues/24). +- @tomer953 removed a duplicate flag to QueryFlags. +- @Ebsan fixed an issue with downloading cross-platform extensions [#24](https://github.com/LOLINTERNETZ/vscodeoffline/issues/24). ## [1.0.20] + ### Fixed - - Fixed an issue when downloading multiple versions of extensions. Thanks @forky2! - + +- Fixed an issue when downloading multiple versions of extensions. Thanks @forky2! + ## [1.0.19] ### Fixed - - Lots of really solid bug fixes. Thank you to @fullylegit! Resilience improvements when fetching from marketplace. Thanks @forky2 and @ebsan. +- Lots of really solid bug fixes. Thank you to @fullylegit! Resilience improvements when fetching from marketplace. Thanks @forky2 and @ebsan. ## [1.0.18] - + ### Changed - - Meta release to trigger CI. +- Meta release to trigger CI. ## [1.0.17] - + ### Changed - - CORS support for gallery. Thanks @kenyon! - + +- CORS support for gallery. Thanks @kenyon! + ## [1.0.16] ### Changed - - Support for saving sync logs to file. Thanks @ap0yuv! +- Support for saving sync logs to file. Thanks @ap0yuv! ## [1.0.16] ### Changed - - Improve extension stats handling. - +- Improve extension stats handling. + ## [1.0.14] ### Fixed - - Fixed insider builds being re-fetched. - +- Fixed insider builds being re-fetched. + ## [1.0.13] - + ### Added - - Added initial support for extension version handling. Hopefully this resolves issue #4. +- Added initial support for extension version handling. Hopefully this resolves issue #4. ## [1.0.12] - -### Fixed - - @ttutko fixed a bug preventing multiple build qualities (stable/insider) from being downloaded. Thanks @darkonejr for investigating and reporting. - +### Fixed + +- @ttutko fixed a bug preventing multiple build qualities (stable/insider) from being downloaded. Thanks @darkonejr for investigating and reporting. + ## [1.0.11] ### Fixed - - Fixed bugs in Gallery sorting, and added timeouts for Sync. - - + +- Fixed bugs in Gallery sorting, and added timeouts for Sync. + ## [1.0.10] ### Changed - - Refactored to improve consistency. +- Refactored to improve consistency. ## [1.0.9] - + ### Added - - Added support for Remote Development, currently (2019-05-12) available to insiders. Refactored various badness. +- Added support for Remote Development, currently (2019-05-12) available to insiders. Refactored various badness. ## [1.0.8] - + ### Added - - Insiders support and extension packs (remotes). + +- Insiders support and extension packs (remotes). diff --git a/docker-compose.yml b/docker-compose.yml index 39acb4a..1129444 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,10 +23,10 @@ services: image: lolinternet/vscgallery:latest build: context: ./ - dockerfile: ./vscoffline/vscgallery/Dockerfile + dockerfile: ./vscoffline/vscgallery/Dockerfile volumes: # Enable for dev - #- ./vscoffline/:/opt/vscoffline/:ro # Enable for dev + #- ./vscoffline/:/opt/vscoffline/:ro - ./artifacts/:/artifacts/ # Enable for custom SSL certs #- ./vscoffline/vscgallery/ssl/:/opt/vscoffline/vscgallery/ssl # Enable for custom SSL certs diff --git a/vscoffline/server.py b/vscoffline/server.py index 51dd815..3ffbb2f 100644 --- a/vscoffline/server.py +++ b/vscoffline/server.py @@ -1,7 +1,7 @@ import os, sys, time, json, glob import falcon from distutils.version import LooseVersion -from logzero import logger as log +import logging as log from wsgiref import simple_server from watchdog.observers.polling import PollingObserver from watchdog.events import FileSystemEventHandler @@ -164,7 +164,7 @@ def process_loaded_extension(self, extension, extensiondir): if "targetPlatform" in version: targetPlatform = version['targetPlatform'] asseturi = vsc.URLROOT + os.path.join(extensiondir, version['version'], targetPlatform) - else: + else: asseturi = vsc.URLROOT + os.path.join(extensiondir, version['version']) version['assetUri'] = asseturi @@ -427,5 +427,10 @@ def on_modified(self, event): application.add_static_route('/artifacts/', '/artifacts/') if __name__ == '__main__': + log.basicConfig( + format='[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d] %(message)s', + datefmt='%y%m%d %H:%M:%S', + level=log.DEBUG + ) httpd = simple_server.make_server('0.0.0.0', 5000, application) httpd.serve_forever() diff --git a/vscoffline/sync.py b/vscoffline/sync.py index 46e81b2..284012c 100644 --- a/vscoffline/sync.py +++ b/vscoffline/sync.py @@ -6,17 +6,17 @@ import pathlib import hashlib import uuid -import logzero import logging import json import time import datetime from typing import List from platform import release -from logzero import logger as log +import logging as log from pytimeparse.timeparse import timeparse import vsc from distutils.dir_util import create_tree +from requests.adapters import HTTPAdapter, Retry class VSCUpdateDefinition(object): @@ -225,7 +225,7 @@ def __init__(self, identity, raw=None): if 'extensionId' in raw: self.extensionId = raw['extensionId'] - def download_assets(self, destination): + def download_assets(self, destination, session): for version in self.versions: targetplatform = '' if "targetPlatform" in version: @@ -242,12 +242,21 @@ def download_assets(self, destination): destfile = os.path.join(ver_destination, f'{asset}') create_tree(os.path.abspath(os.sep), (destfile,)) if not os.path.exists(destfile): - log.debug( - f'Downloading {self.identity} {asset} to {destfile}') - result = requests.get( - url, allow_redirects=True, timeout=vsc.TIMEOUT) - with open(destfile, 'wb') as dest: - dest.write(result.content) + for i in range(5): + try: + if i == 0: + log.debug(f'Downloading {self.identity} {asset} to {destfile}') + else: + log.info(f'Retrying {i+1}, download {self.identity} {asset} to {destfile}') + result = session.get( + url, allow_redirects=True, timeout=vsc.TIMEOUT) + with open(destfile, 'wb') as dest: + dest.write(result.content) + break + except requests.exceptions.ProxyError: + log.info("ProxyError: Retrying.") + except requests.exceptions.ReadTimeout: + log.info("ReadTimeout: Retrying.") def process_embedded_extensions(self, destination, mp): """ @@ -354,10 +363,11 @@ def signal_updated(artifactdir): class VSCMarketplace(object): - def __init__(self, insider, prerelease, version): + def __init__(self, insider, prerelease, version, session): self.insider = insider self.prerelease = prerelease self.version = version + self.session = session def get_recommendations(self, destination, totalrecommended): recommendations = self.search_top_n(totalrecommended) @@ -389,7 +399,7 @@ def get_recommendations(self, destination, totalrecommended): return recommendations def get_recommendations_old(self, destination): - result = requests.get(vsc.URL_RECOMMENDATIONS, + result = self.session.get(vsc.URL_RECOMMENDATIONS, allow_redirects=True, timeout=vsc.TIMEOUT) if result.status_code != 200: log.warning( @@ -409,7 +419,7 @@ def get_recommendations_old(self, destination): return packages def get_malicious(self, destination, extensions=None): - result = requests.get( + result = self.session.get( vsc.URL_MALICIOUS, allow_redirects=True, timeout=vsc.TIMEOUT) if result.status_code != 200: log.warning( @@ -526,7 +536,7 @@ def _query_marketplace(self, filtertype, filtervalue, pageNumber=0, pageSize=500 log.info("Retrying pull page %d attempt %d." % (pageNumber, i+1)) try: - result = requests.post(vsc.URL_MARKETPLACEQUERY, headers=self._headers( + result = self.session.post(vsc.URL_MARKETPLACEQUERY, headers=self._headers( ), json=query, allow_redirects=True, timeout=vsc.TIMEOUT) if result: break @@ -664,16 +674,22 @@ def __repr__(self): config = parser.parse_args() if config.debug: - logzero.loglevel(logging.DEBUG) + loglevel = logging.DEBUG else: - logzero.loglevel(logging.INFO) + loglevel = logging.INFO if config.logfile: log_dir = os.path.dirname(os.path.abspath(config.logfile)) if not os.path.exists(log_dir): raise FileNotFoundError( f'Log directory does not exist at {log_dir}') - logzero.logfile(config.logfile, maxBytes=1000000, backupCount=3) + logging.basicConfig(filename=config.logfile, encoding='utf-8', level=loglevel) + else: + log.basicConfig( + format='[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d] %(message)s', + datefmt='%y%m%d %H:%M:%S', + level=loglevel + ) config.artifactdir_installers = os.path.join( os.path.abspath(config.artifactdir), 'installers') @@ -705,11 +721,17 @@ def __repr__(self): if config.frequency: config.frequency = timeparse(config.frequency) + session = requests.Session() + retries = Retry(total=5, + backoff_factor=0.1, + status_forcelist=[ 500, 502, 503, 504 ]) + session.mount('https://', HTTPAdapter(max_retries=retries)) + while True: versions = [] extensions = {} mp = VSCMarketplace(config.checkinsider, - config.prerelease, config.version) + config.prerelease, config.version, session) if config.checkbinaries and not config.skipbinaries: log.info('Syncing VS Code Update Versions') @@ -776,7 +798,7 @@ def __repr__(self): log.info( f'Progress {count}/{len(extensions)} ({count/len(extensions)*100:.1f}%)') extensions[identity].download_assets( - config.artifactdir_extensions) + config.artifactdir_extensions, session) bonus = extensions[identity].process_embedded_extensions( config.artifactdir_extensions, mp) + bonus extensions[identity].save_state(config.artifactdir_extensions) @@ -784,7 +806,7 @@ def __repr__(self): for bonusextension in bonus: log.debug(f'Processing Embedded Extension: {bonusextension}') - bonusextension.download_assets(config.artifactdir_extensions) + bonusextension.download_assets(config.artifactdir_extensions, session) bonusextension.save_state(config.artifactdir_extensions) log.info('Complete') diff --git a/vscoffline/vsc.py b/vscoffline/vsc.py index 24d4a9d..b5e6d69 100644 --- a/vscoffline/vsc.py +++ b/vscoffline/vsc.py @@ -5,8 +5,7 @@ import pathlib from enum import IntFlag from typing import Any, Dict, List, Union - -from logzero import logger as log +import logging as log PLATFORMS = ["win32", "linux", "linux-deb", "linux-rpm", "darwin", "linux-snap", "server-linux"] ARCHITECTURES = ["", "x64"] @@ -156,11 +155,16 @@ def first_file(filepath: Union[str, pathlib.Path], pattern: str, reverse: bool = @staticmethod def folders_in_folder(filepath: str) -> List[str]: - return [f for f in os.listdir(filepath) if os.path.isdir(os.path.join(filepath, f))] + listing = [f for f in os.listdir(filepath) if os.path.isdir(os.path.join(filepath, f))] + listing.sort() + return listing @staticmethod def files_in_folder(filepath: str) -> List[str]: - return [f for f in os.listdir(filepath) if os.path.isfile(os.path.join(filepath, f))] + listing = [f for f in os.listdir(filepath) if os.path.isfile(os.path.join(filepath, f))] + listing.sort() + return listing + @staticmethod def seconds_to_human_time(seconds: int) -> str: diff --git a/vscoffline/vscgallery/requirements.txt b/vscoffline/vscgallery/requirements.txt index a6c7466..6bac07d 100644 --- a/vscoffline/vscgallery/requirements.txt +++ b/vscoffline/vscgallery/requirements.txt @@ -1,4 +1,3 @@ -logzero requests falcon gunicorn diff --git a/vscoffline/vscsync/requirements.txt b/vscoffline/vscsync/requirements.txt index 0203ba7..1d1c434 100644 --- a/vscoffline/vscsync/requirements.txt +++ b/vscoffline/vscsync/requirements.txt @@ -1,3 +1,2 @@ -logzero requests pytimeparse \ No newline at end of file