🙌 Fix git-svn clone

.github/workflows/main.yml

@@ -21,10 +21,16 @@ jobs:
                   java-version: '11.x'
             - name: 🪦 Install node.js packages
               run: npm install
+            - name: 🚮 Remove current git
+              run: sudo apt-get remove git git-man
+            - name: Add repo
+              run: sudo add-apt-repository --remove --yes ppa:git-core/ppa
             - name: Update platform
               run: sudo apt-get update
+            - name: Upgrade platform
+              run: sudo apt-get upgrade
             - name: Install git-svn
-              run: sudo apt-get install git-svn
+              run: sudo apt-get install --yes git git-svn
             - name: 🔍 Analyze code with SonarQube
               env:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -44,3 +50,5 @@ jobs:
                   GIT_TAG=:${GITHUB_REF#refs/tags/}
                   DOCKER_TAG=${GIT_TAG#:refs/heads/main}
                   ./mvnw -ntp jib:build -Djib.to.image=yodamad/svn2git:latest -Djib.to.auth.username="${{ secrets.DOCKERHUB_LOGIN }}" -Djib.to.auth.password="${{ secrets.DOCKERHUB_PWD }}"
+                  VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)
+                  ./mvnw -ntp jib:build -Djib.to.image=yodamad/svn2git:$VERSION -Djib.to.auth.username="${{ secrets.DOCKERHUB_LOGIN }}" -Djib.to.auth.password="${{ secrets.DOCKERHUB_PWD }}"

pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>fr.yodamad.svn2git</groupId>
     <artifactId>svn-2-git</artifactId>
-    <version>2.1.0</version>
+    <version>2.1.1</version>
     <packaging>jar</packaging>
     <name>Svn 2 GitLab</name>
 

src/main/java/fr/yodamad/svn2git/config/ApplicationProperties.java

@@ -128,6 +128,11 @@ public static class Svn {
          */
         public String svnUrlModifiable;
 
+        /**
+         * Max attempts of git-svn fetch before failing migration
+         */
+        public Integer maxFetchAttempts;
+
         public String getUser() {
             return user;
         }
@@ -167,6 +172,10 @@ public String getSvnUrlModifiable() {
         public void setSvnUrlModifiable(String svnUrlModifiable) {
             this.svnUrlModifiable = svnUrlModifiable;
         }
+
+        public Integer getMaxFetchAttempts() { return maxFetchAttempts; }
+
+        public void setMaxFetchAttempts(Integer maxFetchAttempts) { this.maxFetchAttempts = maxFetchAttempts; }
     }
 
     public static class Gitlab {

src/main/java/fr/yodamad/svn2git/domain/Migration.java

@@ -146,6 +146,10 @@ public class Migration implements Serializable {
     @JsonView(View.Public.class)
     private String uploadType;
 
+    @Column(name = "emptydirs")
+    @JsonView(View.Public.class)
+    private Boolean emptyDirs = false;
+
     @OneToMany(mappedBy = "migration")
     @OrderBy("id ASC")
     private Set<MigrationHistory> histories = new HashSet<>();
@@ -530,6 +534,10 @@ public void setFlat(Boolean flat) {
 
     public void setUploadType(String uploadType) { this.uploadType = uploadType; }
 
+    public Boolean getEmptyDirs() { return emptyDirs; }
+
+    public void setEmptyDirs(Boolean emptyDirs) { this.emptyDirs = emptyDirs; }
+
     // jhipster-needle-entity-add-getters-setters - JHipster will add getters and setters here, do not remove
 
     @Override

src/main/java/fr/yodamad/svn2git/domain/enumeration/StepEnum.java

@@ -7,5 +7,5 @@ public enum StepEnum {
     GIT_CLONE, GITLAB_PROJECT_CREATION, SVN_CHECKOUT, GIT_CLEANING, GIT_PUSH, CLEANING, GIT_MV, INIT, BRANCH_CLEAN,
     TAG_CLEAN, README_MD, GIT_CONFIG_GC_AUTO_OFF, GIT_CONFIG_GLOBAL_GC_AUTO_OFF, GIT_GC_EXPLICIT,
     GIT_DYNAMIC_LOCAL_CONFIG, GIT_SHOW_CONFIG, LIST_REMOVED_FILES, UPLOAD_TO_ARTIFACTORY, ARTIFACTORY_FOLDER_CLEANING,
-    SVN_COPY_ROOT_FOLDER, ULIMIT, GIT_SET_CONFIG, UPLOAD_TO_GITLAB, UPLOAD_TO_NEXUS
+    SVN_COPY_ROOT_FOLDER, ULIMIT, GIT_SET_CONFIG, UPLOAD_TO_GITLAB, UPLOAD_TO_NEXUS, SVN_FETCH, GIT_GC
 }

src/main/kotlin/fr/yodamad/svn2git/service/GitManager.kt

@@ -96,10 +96,76 @@ open class GitManager(val historyMgr: HistoryManager,
         val history = historyMgr.startStep(workUnit.migration, StepEnum.SVN_CHECKOUT,
             (if (workUnit.commandManager.isFirstAttemptMigration) "" else Constants.REEXECUTION_SKIPPING) + safeCommand)
         // Only Clone if first attempt at migration
+        var cloneOK = true
         if (workUnit.commandManager.isFirstAttemptMigration) {
-            execCommand(workUnit.commandManager, workUnit.root, cloneCommand, safeCommand)
+            try {
+                execCommand(workUnit.commandManager, workUnit.root, cloneCommand, safeCommand)
+            } catch (thr: Throwable) {
+                LOG.warn("Cannot git svn clone", thr.message)
+                cloneOK = false
+                var round = 0
+                var notOk = true
+                while (round++ < applicationProperties.svn.maxFetchAttempts && notOk) {
+                    notOk = gitSvnFetch(workUnit, round)
+                    gitGC(workUnit, round)
+                }
+            }
+        }
+        if (cloneOK) {
+            historyMgr.endStep(history, StatusEnum.DONE, null)
+        } else {
+            historyMgr.endStep(history, StatusEnum.DONE_WITH_WARNINGS, null)
+        }
+    }
+
+    /**
+     * Git svn fetch command to copy svn as git repository
+     *
+     * @param workUnit Current work unit
+     * @param round Round number
+     * @return if fetch is in failure or not
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Throws(IOException::class, InterruptedException::class)
+    open fun gitSvnFetch(workUnit: WorkUnit, round: Int) : Boolean {
+        val fetchCommand = "git svn fetch";
+
+        val history = historyMgr.startStep(workUnit.migration, StepEnum.SVN_FETCH, "Round $round : $fetchCommand")
+        return try {
+            execCommand(workUnit.commandManager, workUnit.directory, fetchCommand)
+            historyMgr.endStep(history, StatusEnum.DONE, null)
+            false
+        } catch (thr: Throwable) {
+            LOG.error("Cannot git svn fetch", thr.printStackTrace())
+            historyMgr.endStep(history, StatusEnum.FAILED, null)
+            true
+        }
+    }
+
+    /**
+     * Git gc command to cleanup unnecessary files and optimize the local repository
+     *
+     * @param workUnit Current work unit
+     * @param round Round number
+     * @return if fetch is in failure or not
+     * @throws IOException
+     * @throws InterruptedException
+     */
+    @Throws(IOException::class, InterruptedException::class)
+    open fun gitGC(workUnit: WorkUnit, round: Int) : Boolean {
+        val gcCommand = "git gc";
+
+        val history = historyMgr.startStep(workUnit.migration, StepEnum.GIT_GC, "Round $round : $gcCommand")
+        return try {
+            execCommand(workUnit.commandManager, workUnit.directory, gcCommand)
+            historyMgr.endStep(history, StatusEnum.DONE, null)
+            false
+        } catch (thr: Throwable) {
+            LOG.error("Cannot git gc", thr.printStackTrace())
+            historyMgr.endStep(history, StatusEnum.FAILED, null)
+            true
         }
-        historyMgr.endStep(history, StatusEnum.DONE, null)
     }
 
     /**

src/main/kotlin/fr/yodamad/svn2git/service/util/GitCommandManager.kt

@@ -47,7 +47,8 @@ open class GitCommandManager(val historyMgr: HistoryManager,
             setSvnElement("branches", workUnit.migration.branches, workUnit),
             setSvnElement("tags", workUnit.migration.tags, workUnit),
             ignorePaths, ignoreRefs,
-            if (applicationProperties.getFlags().isGitSvnClonePreserveEmptyDirsOption) "--preserve-empty-dirs" else EMPTY,
+            if (workUnit.migration.emptyDirs) "--preserve-empty-dirs"
+            else if (workUnit.migration.emptyDirs == null && applicationProperties.getFlags().isGitSvnClonePreserveEmptyDirsOption) "--preserve-empty-dirs" else EMPTY,
             if (workUnit.migration.svnUrl.endsWith("/")) workUnit.migration.svnUrl else "${workUnit.migration.svnUrl}/",
             workUnit.migration.svnGroup)
 

src/main/kotlin/fr/yodamad/svn2git/web/rest/GitlabResource.kt

@@ -103,7 +103,7 @@ open class GitlabResource(val gitlabAdmin: GitlabAdmin,
                             .findAny()
                         if (subgroup.isPresent) {
                             if (cycle == depth - 1) {
-                                return checkUserForProject(user!!, group, gitlab)
+                                return checkUserForProject(user!!, subgroup, gitlab)
                             }
                             cycle++
                             groupId = subgroup.get().id

src/main/resources/config/application.yml

@@ -155,6 +155,7 @@ application:
         password:
         credentials: required
         svnUrlModifiable: true
+        maxFetchAttempts: 20
     override:
         extensions: false
         mappings: false

src/main/resources/config/liquibase/changelog/2021031721150000_added_Migration_emptydirs_column.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<databaseChangeLog
+    xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+    xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd
+                        http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
+
+    <property name="now" value="now()" dbms="h2"/>
+    <property name="now" value="now()" dbms="mysql"/>
+    <property name="autoIncrement" value="true"/>
+
+    <changeSet id="emptydirs" author="mvt">
+
+        <addColumn tableName="migration">
+            <column name="emptydirs" type="boolean">
+                <constraints nullable="true" />
+            </column>
+        </addColumn>
+
+    </changeSet>
+
+</databaseChangeLog>

src/main/resources/config/liquibase/master.xml

@@ -27,5 +27,6 @@
     <include file="config/liquibase/changelog/2020101217150000_added_Migration_flat_column.xml" relativeToChangelogFile="false"/>
     <include file="config/liquibase/changelog/2021022821550000_added_Migration_gitlab_project_id.xml" relativeToChangelogFile="false"/>
     <include file="config/liquibase/changelog/2021030122350000_added_Migration_upload_type.xml" relativeToChangelogFile="false"/>
+    <include file="config/liquibase/changelog/2021031721150000_added_Migration_emptydirs_column.xml" relativeToChangelogFile="false"/>
 
 </databaseChangeLog>

src/main/webapp/app/entities/migration/migration-update.component.html

@@ -165,7 +165,7 @@ <h2 id="jhi-migration-heading" jhiTranslate="svn2GitApp.migration.home.createOrE
                 <div class="form-group">
                     <label class="form-control-label" jhiTranslate="svn2GitApp.migration.preserveEmptyDirs" for="field_preserveEmptyDirs">Preserve Empty Dirs</label>
                     <input type="checkbox" class="form-control" name="preserveEmptyDirs" id="field_preserveEmptyDirs"
-                        [(ngModel)]="migration.preserveEmptyDirs" />
+                        [(ngModel)]="migration.emptyDirs" />
                 </div>
 
             </div>

src/main/webapp/app/migration/migration-stepper.component.ts

@@ -573,9 +573,9 @@ export class MigrationStepperComponent implements OnInit {
             console.log(this.mig.maxFileSize);
         }
         if (this.preserveEmptyDirs !== undefined) {
-            this.mig.preserveEmptyDirs = this.preserveEmptyDirs;
+            this.mig.emptyDirs = this.preserveEmptyDirs;
         } else {
-            this.mig.preserveEmptyDirs = false;
+            this.mig.emptyDirs = false;
         }
         if (this.staticExtensions !== undefined && this.staticExtensions.length > 0) {
             const values: string[] = [];

src/main/webapp/app/shared/model/migration.model.ts

@@ -40,7 +40,7 @@ export interface IMigration {
     branchesToMigrate?: string;
     createdTimestamp?: Moment;
     workingDirectory?: string;
-    preserveEmptyDirs?: boolean;
+    emptyDirs?: boolean;
     histories?: IMigrationHistory[];
     mappings?: IMapping[];
     flat?: boolean;
@@ -72,12 +72,12 @@ export class Migration implements IMigration {
         public branchesToMigrate?: string,
         public createdTimestamp?: Moment,
         public workingDirectory?: string,
-        public preserveEmptyDirs?: boolean,
+        public emptyDirs?: boolean,
         public histories?: IMigrationHistory[],
         public mappings?: IMapping[],
         public flat?: boolean,
         public uploadType?: string
     ) {
-        this.preserveEmptyDirs = this.preserveEmptyDirs || false;
+        this.emptyDirs = this.emptyDirs || false;
     }
 }