[sos] Add 'upload' component to upload existing reports and files

This commit marks the beginning of the addition of a new 'upload' component for sos, which can be used to upload already created sos reports, collects, or other files like logs or vmcores to a policy defined location. The user needs to specify a file location, and can make use of any of the options that exist nowadays for the --upload option. This first commit includes: - The initial framework for the 'upload' component. - The new man page for 'sos upload'. - The code in the component 'help' to show information about the component. - The code in sos/__init__.py to deal with the component. - The code for uploads to Red Hat and Ubuntu systems. - The code to allow uploads specifying remote destination, called profiles in this implementation. For example, you could generate a sos report in a CentOS system and specify a profile 'redhat' to upload to the Red Hat Customer Portal. - And modifications to setup.py to build the man pages. Related: RHEL-23032, SUPDEV-138, CLIOT-481 Signed-off-by: Jose Castillo <[email protected]>
sosreport · Jan 2, 2025 · 68d9933 · 68d9933
1 parent ace5715
commit 68d9933
Show file tree

Hide file tree

Showing 13 changed files with 1,532 additions and 921 deletions.
diff --git a/man/en/sos-upload.1 b/man/en/sos-upload.1
@@ -0,0 +1,149 @@
+.TH UPLOAD 1 "December 2024"
+
+.SH NAME
+sos_upload \- Upload files like previously generated sos reports or logs to a policy specific location
+.SH SYNOPSIS
+.B sos upload FILE [options]
+          [--case-id id]\fR
+          [--upload-url url]\fR
+          [--upload-user user]\fR
+          [--upload-pass pass]\fR
+          [--upload-directory dir]\fR
+          [--upload-method]\fR
+          [--upload-no-ssl-verify]\fR
+          [--upload-protocol protocol]\fR
+          [--upload-profile profile]\fR
+          [--upload-s3-endpoint  endpoint]\fR
+          [--upload-s3-region region]\fR
+          [--upload-s3-bucket bucket]\fR
+          [--upload-s3-access-key key]\fR
+          [--upload-s3-secret-key key]\fR
+          [--upload-s3-object-prefix prefix]\fR
+
+.PP
+.SH DESCRIPTION
+upload is an sos subcommand to upload sos reports, logs, vmcores, or other files to a policy defined remote location, or a user defined one.
+.SH REQUIRED ARGUMENTS
+.B FILE
+.TP
+The path to the archive that is to be uploaded.
+.SH OPTIONS
+.TP
+.B \--case-id NUMBER
+Specify a case identifier to associate with the archive.
+Identifiers may include alphanumeric characters, commas and periods ('.').
+.TP
+.B \--upload-url URL
+If a vendor does not provide a default upload location, or if you would like to upload
+the archive to a different location, specify the address here.
+
+An upload protocol MUST be specified in this URL. Currently uploading is supported
+for HTTPS, SFTP, and FTP protocols.
+
+If your destination server listens on a non-standard port, specify the listening
+port in the URL.
+.TP 
+.B \-\-upload-user USER
+If a vendor does not provide a default user for uploading, specify the username here.
+
+If --batch is used and this option is omitted, no username will
+be collected and thus uploads will fail if no vendor default is set.
+
+You also have the option of providing this value via the SOSUPLOADUSER environment
+variable. If this variable is set, then no username prompt will occur and --batch
+may be used provided all other required values (case number, upload password)
+are provided.   
+.TP       
+.B \-\-upload-pass PASS 
+Specify the password to use for authentication with the destination server.
+
+If this option is omitted and upload is requested, you will be prompted for one.
+
+If --batch is used, this prompt will not occur, so any uploads are likely to fail unless
+this option is used.
+
+Note that this will result in the plaintext string appearing in `ps` output that may
+be collected by sos and be in the archive. If a password must be provided by you
+for uploading, it is strongly recommended to not use --batch and enter the password
+when prompted rather than using this option.
+
+You also have the option of providing this value via the SOSUPLOADPASSWORD environment
+variable. If this variable is set, then no password prompt will occur and --batch may
+be used provided all other required values (case number, upload user) are provided.
+.TP       
+.B \--upload-directory DIR
+Specify a directory to upload to, if one is not specified by a vendor default location
+or if your destination server does not allow writes to '/'.
+.TP       
+.B \--upload-method METHOD
+Specify the HTTP method to use for uploading to the provided --upload-url. Valid
+values are 'auto' (default), 'put', or 'post'. The use of 'auto' will default to
+the method required by the policy-default upload location, if one exists.
+
+This option has no effect on upload protocols other than HTTPS.
+.TP
+.B \--upload-no-ssl-verify
+Disable SSL verification for HTTPS uploads. This may be used to allow uploading
+to locations that have self-signed certificates, or certificates that are otherwise
+untrusted by the local system.
+
+Default behavior is to perform SSL verification against all upload locations.
+.TP
+.B \--upload-protocol PROTO
+Manually specify the protocol to use for uploading to the target \fBupload-url\fR.
+
+Normally this is determined via the upload address, assuming that the protocol is part
+of the address provided, e.g. 'https://example.com'. By using this option, sos will skip
+the protocol check and use the method defined for the specified PROTO.
+
+For RHEL systems, setting this option to \fBsftp\fR will skip the initial attempt to
+upload to the Red Hat Customer Portal, and only attempt an upload to Red Hat's SFTP server,
+which is typically used as a fallback target.
+
+Valid values for PROTO are: 'auto' (default), 'https', 'ftp', 'sftp', 's3'.
+
+.TP
+.B \--upload-profile PROFILE
+Manually specify the upload profile to use for uploading. Possible profiles
+are: redhat, ubuntu, local. 
+sos to upload a file to a vendor-specific location explicitly.
+If this option is not called, sos will try to determine the local profile 
+and use it for uploads.
+
+.TP
+.B \---upload-s3-endpoint ENDPOINT
+Endpoint to upload to for S3 bucket
+
+.TP
+.B \---upload-s3-region REGION
+Region to upload to for S3 bucket
+
+.TP
+.B \---upload-s3-bucket BUCKET
+Name of the S3 bucket to upload to
+
+.TP
+.B \---upload-s3-access-key KEY
+Access key for the S3 bucket
+
+.TP
+.B \-  --upload-s3-secret-key KEY
+Secret key for the S3 bucket
+
+.TP
+.B \-  --upload-s3-object-prefix PREFIX
+Prefix for the S3 object/key
+
+.SH SEE ALSO
+.BR sos (1)
+.BR sos-report (1)
+.BR sos-clean (1)
+.BR sos.conf (5)
+.BR sos-collect (1)
+
+.SH MAINTAINER
+.nf
+Maintained on GitHub at https://github.com/sosreport/sos
+.fi
+.SH AUTHORS & CONTRIBUTORS
+See \fBAUTHORS\fR file in the package documentation.
diff --git a/setup.py b/setup.py
@@ -24,7 +24,8 @@
         ('share/man/man1', ['man/en/sosreport.1', 'man/en/sos-report.1',
                             'man/en/sos.1', 'man/en/sos-collect.1',
                             'man/en/sos-collector.1', 'man/en/sos-clean.1',
-                            'man/en/sos-mask.1', 'man/en/sos-help.1']),
+                            'man/en/sos-mask.1', 'man/en/sos-help.1',
+                            'man/en/sos-upload.1']),
         ('share/man/man5', ['man/en/sos.conf.5']),
         ('share/licenses/sos', ['LICENSE']),
         ('share/doc/sos', ['AUTHORS', 'README.md']),

diff --git a/sos/__init__.py b/sos/__init__.py
@@ -54,10 +54,12 @@ def __init__(self, args):
         import sos.report
         import sos.cleaner
         import sos.help
+        import sos.upload
         self._components = {
             'report': (sos.report.SoSReport, ['rep']),
             'clean': (sos.cleaner.SoSCleaner, ['cleaner', 'mask']),
-            'help': (sos.help.SoSHelper, [])
+            'help': (sos.help.SoSHelper, []),
+            'upload': (sos.upload.SoSUpload, [])
         }
         # some distros do not want pexpect as a default dep, so try to load
         # collector here, and if it fails add an entry that implies it is at

diff --git a/sos/collector/__init__.py b/sos/collector/__init__.py
@@ -33,6 +33,7 @@
 from sos.component import SoSComponent
 from sos.utilities import bold
 from sos import __version__
+from sos.upload import SoSUpload
 
 COLLECTOR_CONFIG_DIR = '/etc/sos/groups.d'
 
@@ -145,7 +146,8 @@ class SoSCollector(SoSComponent):
         'upload_s3_bucket': None,
         'upload_s3_access_key': None,
         'upload_s3_secret_key': None,
-        'upload_s3_object_prefix': None
+        'upload_s3_object_prefix': None,
+        'upload_profile': None
     }
 
     def __init__(self, parser, parsed_args, cmdline_args):
@@ -1204,14 +1206,6 @@ def execute(self):
 
         self.intro()
 
-        if self.opts.batch and self.opts.password:
-            self.exit(
-                "\nsos-collector was called with incompatible options --batch "
-                "and --password.\nIf you need to use --password,"
-                " please omit batch mode.\n",
-                1
-            )
-
         self.configure_sos_cmd()
         self.prep()
         self.display_nodes()
@@ -1304,10 +1298,23 @@ def collect(self):
             msg = 'No sos reports were collected, nothing to archive...'
             self.exit(msg, 1)
 
-        if (self.opts.upload and self.policy.get_upload_url()) or \
+        if self.opts.upload or \
                 self.opts.upload_s3_endpoint:
             try:
-                self.policy.upload_archive(self.arc_name)
+                hook_commons = {
+                    'policy': self.policy,
+                    'tmpdir': self.tmpdir,
+                    'sys_tmp': self.sys_tmp,
+                    'options': self.opts,
+                    'manifest': self.manifest
+                }
+                uploader = SoSUpload(parser=self.parser,
+                                     args=self.args,
+                                     cmdline=self.cmdline,
+                                     in_place=True,
+                                     hook_commons=hook_commons,
+                                     archive=self.arc_name)
+                uploader.execute()
                 self.ui_log.info("Uploaded archive successfully")
             except Exception as err:
                 self.ui_log.error(f"Upload attempt failed: {err}")

diff --git a/sos/help/__init__.py b/sos/help/__init__.py
@@ -100,7 +100,8 @@ def get_obj_for_topic(self):
             'collector': 'SoSCollector',
             'collector.transports': 'RemoteTransport',
             'collector.clusters': 'Cluster',
-            'policies': 'Policy'
+            'policies': 'Policy',
+            'upload': 'SoSUpload'
         }
 
         cls = None
@@ -206,7 +207,8 @@ def display_self_help(self):
             'report.plugins.$plugin': 'Information on a specific $plugin',
             'clean':    'Detailed help on the clean command',
             'collect':  'Detailed help on the collect command',
-            'policies': 'How sos operates on different distributions'
+            'upload': 'Detailed help on the upload command',
+            'policies': 'How sos operates on different distributions',
         }
 
         for sect, value in sections.items():