Skip to content

Commit 5f5b597

Browse files
dormant-userdormant-user
committed
Create a backup approach for GitPython
Make `aws_bucket_name` mandatory Remove empty repositories when cloning wiki fails Bump version to 0.0.2
1 parent 8323ca2 commit 5f5b597

9 files changed

+140
-29
lines changed

‎docs/README.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<html lang="en">
55
<head>
66
<meta charset="utf-8" />
7-
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
7+
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
88

99
<title>Git2S3 &#8212; Git2S3 documentation</title>
1010
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />

‎docs/genindex.html

+7-3
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,17 @@ <h2 id="A">A</h2>
6666
<table style="width: 100%" class="indextable genindextable"><tr>
6767
<td style="width: 33%; vertical-align: top;"><ul>
6868
<li><a href="index.html#git2s3.config.SourceControl.all">all (git2s3.config.SourceControl attribute)</a>
69+
</li>
70+
<li><a href="index.html#git2s3.squire.archer">archer() (in module git2s3.squire)</a>
6971
</li>
7072
<li><a href="index.html#git2s3.exc.ArchiveError">ArchiveError</a>
7173
</li>
7274
<li><a href="index.html#git2s3.config.EnvConfig.aws_access_key_id">aws_access_key_id (git2s3.config.EnvConfig attribute)</a>
73-
</li>
74-
<li><a href="index.html#git2s3.config.EnvConfig.aws_bucket_name">aws_bucket_name (git2s3.config.EnvConfig attribute)</a>
7575
</li>
7676
</ul></td>
7777
<td style="width: 33%; vertical-align: top;"><ul>
78+
<li><a href="index.html#git2s3.config.EnvConfig.aws_bucket_name">aws_bucket_name (git2s3.config.EnvConfig attribute)</a>
79+
</li>
7880
<li><a href="index.html#git2s3.config.EnvConfig.aws_profile_name">aws_profile_name (git2s3.config.EnvConfig attribute)</a>
7981
</li>
8082
<li><a href="index.html#git2s3.config.EnvConfig.aws_region_name">aws_region_name (git2s3.config.EnvConfig attribute)</a>
@@ -103,10 +105,12 @@ <h2 id="C">C</h2>
103105
<td style="width: 33%; vertical-align: top;"><ul>
104106
<li><a href="index.html#git2s3.squire.check_file_presence">check_file_presence() (in module git2s3.squire)</a>
105107
</li>
106-
<li><a href="index.html#git2s3.config.DataStore.clone_url">clone_url (git2s3.config.DataStore attribute)</a>
108+
<li><a href="index.html#git2s3.main.Git2S3.cli">cli() (git2s3.main.Git2S3 method)</a>
107109
</li>
108110
</ul></td>
109111
<td style="width: 33%; vertical-align: top;"><ul>
112+
<li><a href="index.html#git2s3.config.DataStore.clone_url">clone_url (git2s3.config.DataStore attribute)</a>
113+
</li>
110114
<li><a href="index.html#git2s3.main.Git2S3.clone_wiki">clone_wiki() (git2s3.main.Git2S3 method)</a>
111115
</li>
112116
<li><a href="index.html#git2s3.main.Git2S3.cloner">cloner() (git2s3.main.Git2S3 method)</a>

‎docs/index.html

+40-3
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<html lang="en">
55
<head>
66
<meta charset="utf-8" />
7-
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
7+
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
88

99
<title>Welcome to Git2S3’s documentation! &#8212; Git2S3 documentation</title>
1010
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
@@ -94,6 +94,26 @@ <h1>Welcome to Git2S3’s documentation!<a class="headerlink" href="#welcome-to-
9494
</dl>
9595
</dd></dl>
9696

97+
<dl class="py method">
98+
<dt class="sig sig-object py" id="git2s3.main.Git2S3.cli">
99+
<span class="sig-name descname"><span class="pre">cli</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">cmd</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">fail</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">bool</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">True</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">int</span></span></span><a class="headerlink" href="#git2s3.main.Git2S3.cli" title="Permalink to this definition"></a></dt>
100+
<dd><p>Runs CLI commands.</p>
101+
<dl class="field-list simple">
102+
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
103+
<dd class="field-odd"><ul class="simple">
104+
<li><p><strong>cmd</strong> – Command to run.</p></li>
105+
<li><p><strong>fail</strong> – Boolean flag to fail on errors.</p></li>
106+
</ul>
107+
</dd>
108+
<dt class="field-even">Returns<span class="colon">:</span></dt>
109+
<dd class="field-even"><p>Return code after running the command.</p>
110+
</dd>
111+
<dt class="field-odd">Return type<span class="colon">:</span></dt>
112+
<dd class="field-odd"><p>int</p>
113+
</dd>
114+
</dl>
115+
</dd></dl>
116+
97117
<dl class="py method">
98118
<dt class="sig sig-object py" id="git2s3.main.Git2S3.get_all">
99119
<span class="sig-name descname"><span class="pre">get_all</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">source</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#git2s3.config.SourceControl" title="git2s3.config.SourceControl"><span class="pre">SourceControl</span></a></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">Generator</span><span class="p"><span class="pre">[</span></span><span class="pre">Dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#git2s3.main.Git2S3.get_all" title="Permalink to this definition"></a></dt>
@@ -110,7 +130,7 @@ <h1>Welcome to Git2S3’s documentation!<a class="headerlink" href="#welcome-to-
110130

111131
<dl class="py method">
112132
<dt class="sig sig-object py" id="git2s3.main.Git2S3.set_pat">
113-
<span class="sig-name descname"><span class="pre">set_pat</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">url</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Union</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">Url</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#git2s3.main.Git2S3.set_pat" title="Permalink to this definition"></a></dt>
133+
<span class="sig-name descname"><span class="pre">set_pat</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">url</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">Union</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">Url</span><span class="p"><span class="pre">]</span></span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">Optional</span><span class="p"><span class="pre">[</span></span><span class="pre">Union</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">Url</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#git2s3.main.Git2S3.set_pat" title="Permalink to this definition"></a></dt>
114134
<dd><p>Creates an authenticated URL by updating the netloc, and sets that as the origin URL.</p>
115135
<dl class="field-list simple">
116136
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
@@ -240,6 +260,23 @@ <h1>Welcome to Git2S3’s documentation!<a class="headerlink" href="#welcome-to-
240260
</section>
241261
<section id="module-git2s3.squire">
242262
<span id="squire"></span><h1>Squire<a class="headerlink" href="#module-git2s3.squire" title="Permalink to this heading"></a></h1>
263+
<dl class="py function">
264+
<dt class="sig sig-object py" id="git2s3.squire.archer">
265+
<span class="sig-prename descclassname"><span class="pre">git2s3.squire.</span></span><span class="sig-name descname"><span class="pre">archer</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">destination</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">None</span></span></span><a class="headerlink" href="#git2s3.squire.archer" title="Permalink to this definition"></a></dt>
266+
<dd><p>Archives a given directory and deletes it while retaining the zipfile.</p>
267+
<dl class="field-list simple">
268+
<dt class="field-odd">Parameters<span class="colon">:</span></dt>
269+
<dd class="field-odd"><p><strong>destination</strong> – Directory path to be archived.</p>
270+
</dd>
271+
<dt class="field-even">Raises<span class="colon">:</span></dt>
272+
<dd class="field-even"><ul class="simple">
273+
<li><p><strong>AssertionError</strong> – </p></li>
274+
<li><p><strong>If zipfile is not present after archiving.</strong> – </p></li>
275+
</ul>
276+
</dd>
277+
</dl>
278+
</dd></dl>
279+
243280
<dl class="py function">
244281
<dt class="sig sig-object py" id="git2s3.squire.env_loader">
245282
<span class="sig-prename descclassname"><span class="pre">git2s3.squire.</span></span><span class="sig-name descname"><span class="pre">env_loader</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">filename</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">os.PathLike</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><a class="reference internal" href="#git2s3.config.EnvConfig" title="git2s3.config.EnvConfig"><span class="pre">EnvConfig</span></a></span></span><a class="headerlink" href="#git2s3.squire.env_loader" title="Permalink to this definition"></a></dt>
@@ -413,7 +450,7 @@ <h1>Configuration<a class="headerlink" href="#configuration" title="Permalink to
413450

414451
<dl class="py attribute">
415452
<dt class="sig sig-object py" id="git2s3.config.EnvConfig.aws_bucket_name">
416-
<span class="sig-name descname"><span class="pre">aws_bucket_name</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></em><a class="headerlink" href="#git2s3.config.EnvConfig.aws_bucket_name" title="Permalink to this definition"></a></dt>
453+
<span class="sig-name descname"><span class="pre">aws_bucket_name</span></span><em class="property"><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="pre">str</span></em><a class="headerlink" href="#git2s3.config.EnvConfig.aws_bucket_name" title="Permalink to this definition"></a></dt>
417454
<dd></dd></dl>
418455

419456
<dl class="py attribute">

‎docs/objects.inv

10 Bytes
Binary file not shown.

‎docs/searchindex.js

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎git2s3/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from git2s3.main import Git2S3
88

9-
version = "0.0.1"
9+
version = "0.0.2"
1010

1111

1212
@click.command()

‎git2s3/config.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ class EnvConfig(BaseSettings):
8686
aws_access_key_id: str | None = None
8787
aws_secret_access_key: str | None = None
8888
aws_region_name: str | None = None
89-
aws_bucket_name: str | None = None
89+
aws_bucket_name: str
9090
aws_s3_prefix: str = f"github_{int(time.time())}"
9191
boto3_retry_attempts: int = 10
9292
boto3_retry_mode: Boto3RetryMode = Boto3RetryMode.standard

‎git2s3/main.py

+73-19
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import logging
22
import os
3-
import secrets
43
import shutil
4+
import subprocess
55
import threading
66
import warnings
77
from collections.abc import Generator
@@ -12,7 +12,7 @@
1212

1313
import git
1414
import requests
15-
from git.exc import GitCommandError
15+
from git.exc import GitCommandError, InvalidGitRepositoryError
1616
from pydantic import HttpUrl
1717

1818
from git2s3 import config, exc, s3, squire
@@ -48,8 +48,14 @@ def __init__(
4848
"X-GitHub-Api-Version": "2022-11-28",
4949
"Content-Type": "application/x-www-form-urlencoded",
5050
}
51-
self.repo = git.Repo()
52-
self.origin = self.repo.remote()
51+
try:
52+
self.repo = git.Repo()
53+
self.origin = self.repo.remote()
54+
except (InvalidGitRepositoryError, ValueError):
55+
self.logger.warning("Unable to use git python, switching to git cli")
56+
self.cli("command -v git") # Make sure git cli works
57+
self.repo = None
58+
self.origin = None
5359
self.clone_dir = os.path.join(os.getcwd(), self.env.git_owner)
5460
warnings.simplefilter("always", exc.DirectoryExists)
5561
warnings.simplefilter("always", exc.UnsupportedSource)
@@ -63,7 +69,8 @@ def __init__(
6369
if profile == "orgs":
6470
if config.SourceControl.gist in self.env.source:
6571
warnings.warn(
66-
f"Gists are not supported for organizations. Removing {config.SourceControl.gist!r} from source.",
72+
"Gists are not supported for organization profiles. "
73+
f"Removing {config.SourceControl.gist.name!r} from source.",
6774
exc.UnsupportedSource,
6875
)
6976
self.env.source.remove(config.SourceControl.gist)
@@ -96,6 +103,35 @@ def profile_type(self) -> str:
96103
f"Failed to get the profile type for {self.env.git_owner}. Please check the owner/organization name."
97104
)
98105

106+
def cli(self, cmd: str, fail: bool = True) -> int:
107+
"""Runs CLI commands.
108+
109+
Args:
110+
cmd: Command to run.
111+
fail: Boolean flag to fail on errors.
112+
113+
Returns:
114+
int:
115+
Return code after running the command.
116+
"""
117+
redacted = cmd.replace(self.env.git_token, "****")
118+
try:
119+
ret_code = subprocess.check_call(
120+
cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, shell=True
121+
)
122+
except subprocess.CalledProcessError as error:
123+
ret_code = error.returncode
124+
if fail:
125+
if error.output:
126+
self.logger.error(error.output)
127+
else:
128+
self.logger.error("Failed to run %s", redacted)
129+
if fail:
130+
assert (
131+
ret_code == 0
132+
), f"{redacted!r} - returned a non-zero exit code: {ret_code}"
133+
return ret_code
134+
99135
def get_all(self, source: config.SourceControl) -> Generator[Dict[str, str]]:
100136
"""Iterate through a target owner/organization to get all available repositories/gists.
101137
@@ -142,7 +178,7 @@ def get_all(self, source: config.SourceControl) -> Generator[Dict[str, str]]:
142178
self.logger.debug("No repos found in page: %d, ending loop.", idx)
143179
break
144180

145-
def set_pat(self, url: str | HttpUrl) -> None:
181+
def set_pat(self, url: str | HttpUrl) -> str | HttpUrl | None:
146182
"""Creates an authenticated URL by updating the netloc, and sets that as the origin URL.
147183
148184
Args:
@@ -164,8 +200,10 @@ def set_pat(self, url: str | HttpUrl) -> None:
164200
url_split.fragment,
165201
)
166202
)
167-
self.origin.config_writer.set("url", joined)
168-
self.origin.config_writer.release()
203+
if self.repo and self.origin:
204+
self.origin.config_writer.set("url", joined)
205+
self.origin.config_writer.release()
206+
return joined
169207

170208
def clone_wiki(self, datastore: config.DataStore) -> None:
171209
"""Clone all the wikis from the repository.
@@ -188,9 +226,25 @@ def clone_wiki(self, datastore: config.DataStore) -> None:
188226
)
189227
if not os.path.isdir(wiki_dest):
190228
os.makedirs(wiki_dest)
229+
wiki_url = self.set_pat(wiki_url)
191230
try:
192-
self.set_pat(wiki_url)
193-
self.repo.clone_from(wiki_url, wiki_dest)
231+
if self.repo and self.origin:
232+
self.repo.clone_from(wiki_url, wiki_dest)
233+
else:
234+
output = self.cli(f"cd {wiki_dest} && git clone {wiki_url}", fail=False)
235+
if output == 0:
236+
try:
237+
squire.archer(wiki_dest)
238+
except AssertionError:
239+
self.logger.error(
240+
"Failed to create a zip file for %s", datastore.name
241+
)
242+
raise exc.ArchiveError(
243+
f"Failed to create a zip file for {datastore.name!r}"
244+
)
245+
else:
246+
# Skip if cloning failed, as wiki pages are not guaranteed to exist
247+
shutil.rmtree(wiki_dest)
194248
except GitCommandError as error:
195249
msg = error.stderr or error.stdout or ""
196250
msg = msg.strip().replace("\n", "").replace("'", "").replace('"', "")
@@ -230,24 +284,24 @@ def worker(self, repo: Dict[str, str]) -> None:
230284
).start()
231285
if not os.path.isdir(repo_dest):
232286
os.makedirs(repo_dest)
287+
datastore.clone_url = self.set_pat(datastore.clone_url)
233288
try:
234-
self.set_pat(datastore.clone_url)
235-
self.repo.clone_from(datastore.clone_url, repo_dest)
289+
if self.repo and self.origin:
290+
self.repo.clone_from(datastore.clone_url, repo_dest)
291+
else:
292+
self.cli(f"cd {repo_dest} && git clone {datastore.clone_url}")
236293
try:
237294
if datastore.description:
238-
desc_file = os.path.join(
239-
repo_dest, f"description_{secrets.token_hex(2)}.txt"
240-
)
295+
desc_file = os.path.join(repo_dest, "description_git2s3.txt")
241296
with open(desc_file, "w") as desc:
242297
desc.write(datastore.description)
243298
desc.flush()
244299
except Exception as warning:
245300
# Adding description file is only an added feature, so no need to fail
246301
self.logger.warning(warning)
247-
shutil.make_archive(repo_dest, "zip", repo_dest)
248-
if os.path.isfile(f"{repo_dest}.zip"):
249-
shutil.rmtree(repo_dest)
250-
else:
302+
try:
303+
squire.archer(repo_dest)
304+
except AssertionError:
251305
self.logger.error("Failed to create a zip file for %s", datastore.name)
252306
raise exc.ArchiveError(
253307
f"Failed to create a zip file for {datastore.name!r}"

‎git2s3/squire.py

+16
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import logging
33
import os
44
import pathlib
5+
import shutil
56
from datetime import datetime
67
from typing import Dict
78

@@ -10,6 +11,21 @@
1011
from git2s3 import config
1112

1213

14+
def archer(destination: str) -> None:
15+
"""Archives a given directory and deletes it while retaining the zipfile.
16+
17+
Args:
18+
destination: Directory path to be archived.
19+
20+
Raises:
21+
AssertionError:
22+
If zipfile is not present after archiving.
23+
"""
24+
shutil.make_archive(destination, "zip", destination)
25+
assert os.path.isfile(f"{destination}.zip")
26+
shutil.rmtree(destination)
27+
28+
1329
def env_loader(filename: str | os.PathLike) -> config.EnvConfig:
1430
"""Loads environment variables based on filetypes.
1531

0 commit comments

Comments
 (0)