Skip to content

Commit

Permalink
improve old updates deletion
Browse files Browse the repository at this point in the history 
support region
fix split file handling
  • Loading branch information
giwty authored and giwty committed Oct 6, 2020
1 parent ba2a8b8 commit 803ad74
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 47 deletions.
2 changes: 1 addition & 1 deletion console.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func (c *Console) Start() {
if settingsObj.OrganizeOptions.DeleteOldUpdateFiles {
progressBar = progressbar.New(2000)
fmt.Printf("\nDeleting old updates\n")
process.DeleteOldUpdates(localDB, c)
process.DeleteOldUpdates(c.baseFolder, localDB, c)
progressBar.Finish()
}

Expand Down
47 changes: 31 additions & 16 deletions db/localSwitchFilesDB.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,14 @@ var (
titleIdRegex = regexp.MustCompile(`\[(?P<titleId>[A-Z,a-z0-9]{16})]`)
)

const (
REASON_UNSUPPORTED_TYPE = iota
REASON_DUPLICATE
REASON_OLD_UPDATE
REASON_UNRECOGNISED
REASON_MALFORMED_FILE
)

type LocalSwitchDBManager struct {
db *bolt.DB
}
Expand Down Expand Up @@ -89,15 +97,21 @@ type SwitchGameFiles struct {
IsSplit bool
}

type SkippedFile struct {
ReasonCode int
ReasonText string
AdditionalInfo string
}

type LocalSwitchFilesDB struct {
TitlesMap map[string]*SwitchGameFiles
Skipped map[ExtendedFileInfo]string
Skipped map[ExtendedFileInfo]SkippedFile
NumFiles int
}

func (ldb *LocalSwitchDBManager) CreateLocalSwitchFilesDB(folders []string, progress ProgressUpdater, recursive bool) (*LocalSwitchFilesDB, error) {
titles := map[string]*SwitchGameFiles{}
skipped := map[ExtendedFileInfo]string{}
skipped := map[ExtendedFileInfo]SkippedFile{}
files := []ExtendedFileInfo{}
for i, folder := range folders {
err := scanFolder(folder, recursive, &files, progress)
Expand Down Expand Up @@ -154,7 +168,7 @@ func scanFolder(folder string, recursive bool, files *[]ExtendedFileInfo, progre
func (ldb *LocalSwitchDBManager) processLocalFiles(files []ExtendedFileInfo,
progress ProgressUpdater,
titles map[string]*SwitchGameFiles,
skipped map[ExtendedFileInfo]string) {
skipped map[ExtendedFileInfo]SkippedFile) {
ind := 0
total := len(files)
for _, file := range files {
Expand Down Expand Up @@ -188,15 +202,15 @@ func (ldb *LocalSwitchDBManager) processLocalFiles(files []ExtendedFileInfo,
!strings.HasSuffix(fileName, "nsp") &&
!strings.HasSuffix(fileName, "nsz") &&
!strings.HasSuffix(fileName, "xcz") {
skipped[file] = "file type is not supported"
skipped[file] = SkippedFile{ReasonCode: REASON_UNSUPPORTED_TYPE, ReasonText: "file type is not supported"}
continue
}

contentMap, err := ldb.getGameMetadata(file, filePath, skipped)

if err != nil {
if _, ok := skipped[file]; !ok {
skipped[file] = "unable to determine title-Id / version - " + err.Error()
skipped[file] = SkippedFile{ReasonText: "unable to determine title-Id / version - " + err.Error(), ReasonCode: REASON_UNRECOGNISED}
}
continue
}
Expand Down Expand Up @@ -224,18 +238,18 @@ func (ldb *LocalSwitchDBManager) processLocalFiles(files []ExtendedFileInfo,
metadata.Type = "Update"

if update, ok := switchTitle.Updates[metadata.Version]; ok {
skipped[file] = "duplicate update file (" + update.ExtendedInfo.Info.Name() + ")"
skipped[file] = SkippedFile{ReasonCode: REASON_DUPLICATE, ReasonText: "duplicate update file (" + update.ExtendedInfo.Info.Name() + ")"}
zap.S().Warnf("-->Duplicate update file found [%v] and [%v]", update.ExtendedInfo.Info.Name(), file.Info.Name())
continue
}
switchTitle.Updates[metadata.Version] = SwitchFileInfo{ExtendedInfo: file, Metadata: metadata}
if metadata.Version > switchTitle.LatestUpdate {
if switchTitle.LatestUpdate != 0 {
skipped[switchTitle.Updates[switchTitle.LatestUpdate].ExtendedInfo] = "old update file, newer update exist locally"
skipped[switchTitle.Updates[switchTitle.LatestUpdate].ExtendedInfo] = SkippedFile{ReasonCode: REASON_OLD_UPDATE, ReasonText: "old update file, newer update exist locally"}
}
switchTitle.LatestUpdate = metadata.Version
} else {
skipped[file] = "old update file, newer update exist locally"
skipped[file] = SkippedFile{ReasonCode: REASON_OLD_UPDATE, ReasonText: "old update file, newer update exist locally"}
}
continue
}
Expand All @@ -244,7 +258,7 @@ func (ldb *LocalSwitchDBManager) processLocalFiles(files []ExtendedFileInfo,
if strings.HasSuffix(metadata.TitleId, "000") {
metadata.Type = "Base"
if switchTitle.BaseExist {
skipped[file] = "duplicate base file (" + switchTitle.File.ExtendedInfo.Info.Name() + ")"
skipped[file] = SkippedFile{ReasonCode: REASON_DUPLICATE, ReasonText: "duplicate base file (" + switchTitle.File.ExtendedInfo.Info.Name() + ")"}
zap.S().Warnf("-->Duplicate base file found [%v] and [%v]", file.Info.Name(), switchTitle.File.ExtendedInfo.Info.Name())
continue
}
Expand All @@ -256,11 +270,11 @@ func (ldb *LocalSwitchDBManager) processLocalFiles(files []ExtendedFileInfo,

if dlc, ok := switchTitle.Dlc[metadata.TitleId]; ok {
if metadata.Version < dlc.Metadata.Version {
skipped[file] = "old DLC file, newer version exist locally"
skipped[file] = SkippedFile{ReasonCode: REASON_OLD_UPDATE, ReasonText: "old DLC file, newer version exist locally"}
zap.S().Warnf("-->Old DLC file found [%v] and [%v]", file.Info.Name(), dlc.ExtendedInfo.Info.Name())
continue
} else if metadata.Version == dlc.Metadata.Version {
skipped[file] = "duplicate DLC file (" + dlc.ExtendedInfo.Info.Name() + ")"
skipped[file] = SkippedFile{ReasonCode: REASON_DUPLICATE, ReasonText: "duplicate DLC file (" + dlc.ExtendedInfo.Info.Name() + ")"}
zap.S().Warnf("-->Duplicate DLC file found [%v] and [%v]", file.Info.Name(), dlc.ExtendedInfo.Info.Name())
continue
}
Expand All @@ -281,14 +295,15 @@ func (ldb *LocalSwitchDBManager) ClearDB() error {
return err
}

func (ldb *LocalSwitchDBManager) getGameMetadata(file ExtendedFileInfo, filePath string, skipped map[ExtendedFileInfo]string) (map[string]*switchfs.ContentMetaAttributes, error) {
func (ldb *LocalSwitchDBManager) getGameMetadata(file ExtendedFileInfo,
filePath string,
skipped map[ExtendedFileInfo]SkippedFile) (map[string]*switchfs.ContentMetaAttributes, error) {

var metadata map[string]*switchfs.ContentMetaAttributes = nil
keys, _ := settings.SwitchKeys()
var err error
fileKey := filePath + "|" + file.Info.Name() + "|" + strconv.Itoa(int(file.Info.Size()))
if keys != nil && keys.GetKey("header_key") != "" {

err = ldb.db.View(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("deep-scan"))
if b == nil {
Expand Down Expand Up @@ -321,20 +336,20 @@ func (ldb *LocalSwitchDBManager) getGameMetadata(file ExtendedFileInfo, filePath
strings.HasSuffix(fileName, "nsz") {
metadata, err = switchfs.ReadNspMetadata(filePath)
if err != nil {
skipped[file] = fmt.Sprintf("failed to read NSP [reason: %v]", err)
skipped[file] = SkippedFile{ReasonCode: REASON_MALFORMED_FILE, ReasonText: fmt.Sprintf("failed to read NSP [reason: %v]", err)}
zap.S().Errorf("[file:%v] failed to read NSP [reason: %v]\n", file.Info.Name(), err)
}
} else if strings.HasSuffix(fileName, "xci") ||
strings.HasSuffix(fileName, "xcz") {
metadata, err = switchfs.ReadXciMetadata(filePath)
if err != nil {
skipped[file] = fmt.Sprintf("failed to read NSP [reason: %v]", err)
skipped[file] = SkippedFile{ReasonCode: REASON_MALFORMED_FILE, ReasonText: fmt.Sprintf("failed to read NSP [reason: %v]", err)}
zap.S().Errorf("[file:%v] failed to read file [reason: %v]\n", file.Info.Name(), err)
}
} else if strings.HasSuffix(fileName, "00") {
metadata, err = fileio.ReadSplitFileMetadata(filePath)
if err != nil {
skipped[file] = fmt.Sprintf("failed to read split files [reason: %v]", err)
skipped[file] = SkippedFile{ReasonCode: REASON_MALFORMED_FILE, ReasonText: fmt.Sprintf("failed to read split files [reason: %v]", err)}
zap.S().Errorf("[file:%v] failed to read NSP [reason: %v]\n", file.Info.Name(), err)
}
}
Expand Down
100 changes: 70 additions & 30 deletions process/organizefolderStructure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,32 +17,41 @@ import (
var (
folderIllegalCharsRegex = regexp.MustCompile(`[/\\?%*:;=|"<>]`)
nonAscii = regexp.MustCompile("[a-zA-Z0-9áéíóú@#%&',.\\s-\\[\\]\\(\\)\\+]")
cjk = regexp.MustCompile("[\u2f70-\u2FA1\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff66-\uff9f\\p{Katakana}\\p{Hiragana}\\p{Hangul}]")
)

func DeleteOldUpdates(localDB *db.LocalSwitchFilesDB, updateProgress db.ProgressUpdater) {
func DeleteOldUpdates(baseFolder string, localDB *db.LocalSwitchFilesDB, updateProgress db.ProgressUpdater) {
i := 0
for k, v := range localDB.TitlesMap {
if !v.BaseExist || v.IsSplit {
continue
for k, v := range localDB.Skipped {
switch v.ReasonCode {
//case db.REASON_DUPLICATE:
case db.REASON_OLD_UPDATE:
fileToRemove := filepath.Join(k.BaseFolder, k.Info.Name())
if updateProgress != nil {
updateProgress.UpdateProgress(0, 0, "deleting "+fileToRemove)
}
zap.S().Infof("Deleting file: %v \n", fileToRemove)
err := os.Remove(fileToRemove)
if err != nil {
zap.S().Errorf("Failed to delete file %v [%v]\n", fileToRemove, err)
continue
}
i++
}
i++

}

if i != 0 && settings.ReadSettings(baseFolder).OrganizeOptions.DeleteEmptyFolders {
if updateProgress != nil {
updateProgress.UpdateProgress(i, len(localDB.TitlesMap), v.File.ExtendedInfo.Info.Name()+k)
updateProgress.UpdateProgress(i, i+1, "deleting empty folders... (can take 1-2min)")
}
if len(v.Updates) > 1 {

for version, update := range v.Updates {
if version < v.LatestUpdate && version != 0 {
fileToRemove := filepath.Join(update.ExtendedInfo.BaseFolder, update.ExtendedInfo.Info.Name())
zap.S().Infof("--> [Delete] Old update file: %v [latest update:%v]\n", fileToRemove, v.LatestUpdate)
err := os.Remove(fileToRemove)
if err != nil {
zap.S().Errorf("Failed to delete file %v [%v]\n", fileToRemove, err)
}
}
}
err := deleteEmptyFolders(baseFolder)
if err != nil {
zap.S().Errorf("Failed to delete empty folders [%v]\n", err)
}
if updateProgress != nil {
updateProgress.UpdateProgress(i+1, i+1, "deleting empty folders... (can take 1-2min)")
}

}
}

Expand All @@ -62,9 +71,10 @@ func OrganizeByFolders(baseFolder string,
tasksSize := len(localDB.TitlesMap) + 2
for k, v := range localDB.TitlesMap {
i++
if v.BaseExist == false {
if !v.BaseExist {
continue
}

if updateProgress != nil {
updateProgress.UpdateProgress(i, tasksSize, v.File.ExtendedInfo.Info.Name())
}
Expand All @@ -77,6 +87,9 @@ func OrganizeByFolders(baseFolder string,
//templateData[settings.TEMPLATE_TYPE] = "BASE"
templateData[settings.TEMPLATE_TITLE_NAME] = titleName
templateData[settings.TEMPLATE_VERSION_TXT] = ""
if _, ok := titlesDB.TitlesMap[k]; ok {
templateData[settings.TEMPLATE_REGION] = titlesDB.TitlesMap[k].Attributes.Region
}
templateData[settings.TEMPLATE_VERSION] = "0"

if v.File.Metadata.Ncap != nil {
Expand All @@ -98,6 +111,29 @@ func OrganizeByFolders(baseFolder string,
}
}

if v.IsSplit {
//in case of a split file, we only rename the folder and then move all the split
//files with the new folder
files, err := ioutil.ReadDir(v.File.ExtendedInfo.BaseFolder)
if err != nil {
continue
}

for _, file := range files {
if _, err := strconv.Atoi(file.Name()[len(file.Name())-1:]); err == nil {
from := filepath.Join(v.File.ExtendedInfo.BaseFolder, file.Name())
to := filepath.Join(destinationPath, file.Name())
err := moveFile(from, to)
if err != nil {
zap.S().Errorf("Failed to move file [%v]\n", err)
continue
}
}
}
continue

}

//process base title
from := filepath.Join(v.File.ExtendedInfo.BaseFolder, v.File.ExtendedInfo.Info.Name())
to := filepath.Join(destinationPath, getFileName(options, v.File.ExtendedInfo.Info.Name(), templateData))
Expand Down Expand Up @@ -210,26 +246,29 @@ func getDlcName(switchTitle *db.SwitchTitle, file db.SwitchFileInfo) string {
}
if dlcAttributes, ok := switchTitle.Dlc[file.Metadata.TitleId]; ok {
name := dlcAttributes.Name
name = strings.ReplaceAll(name, "\n", "")
name = strings.ReplaceAll(name, "\n", " ")
return name
}
return ""
}

func getTitleName(switchTitle *db.SwitchTitle, v *db.SwitchGameFiles) string {
if switchTitle != nil && switchTitle.Attributes.Name != "" {
return switchTitle.Attributes.Name
} else {
res := cjk.FindAllString(switchTitle.Attributes.Name, -1)
if len(res) == 0 {
return switchTitle.Attributes.Name
}
}

if v.File.Metadata.Ncap != nil {
name := v.File.Metadata.Ncap.TitleName["AmericanEnglish"].Title
if name != "" {
return name
}
if v.File.Metadata.Ncap != nil {
name := v.File.Metadata.Ncap.TitleName["AmericanEnglish"].Title
if name != "" {
return name
}
//for non eshop games (cartridge only), grab the name from the file
return db.ParseTitleNameFromFileName(v.File.ExtendedInfo.Info.Name())
}
//for non eshop games (cartridge only), grab the name from the file
return db.ParseTitleNameFromFileName(v.File.ExtendedInfo.Info.Name())

}

func getFolderName(options settings.OrganizeOptions, templateData map[string]string) string {
Expand Down Expand Up @@ -260,6 +299,7 @@ func applyTemplate(templateData map[string]string, useSafeNames bool, template s
result = strings.Replace(result, "{"+settings.TEMPLATE_VERSION+"}", templateData[settings.TEMPLATE_VERSION], 1)
result = strings.Replace(result, "{"+settings.TEMPLATE_TYPE+"}", templateData[settings.TEMPLATE_TYPE], 1)
result = strings.Replace(result, "{"+settings.TEMPLATE_VERSION_TXT+"}", templateData[settings.TEMPLATE_VERSION_TXT], 1)
result = strings.Replace(result, "{"+settings.TEMPLATE_REGION+"}", templateData[settings.TEMPLATE_REGION], 1)
//remove title name from dlc name
dlcName := strings.Replace(templateData[settings.TEMPLATE_DLC_NAME], templateData[settings.TEMPLATE_TITLE_NAME], "", 1)
dlcName = strings.TrimSpace(dlcName)
Expand Down

0 comments on commit 803ad74

Please sign in to comment.