From 7abfde1d98f4cf4553030a96a9fb72873af60330 Mon Sep 17 00:00:00 2001 From: Kaszanas <34846245+Kaszanas@users.noreply.github.com> Date: Mon, 5 May 2025 01:44:22 +0200 Subject: [PATCH 1/9] Bumped Golang version in Dockerfiles --- docker/Dockerfile | 7 ++++--- docker/Dockerfile.dev | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 60ae408..e1c3074 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # All of the maps and mods downloaded when processing SC2ReSet replays: FROM kaszanas/sc2reset_maps_mods:latest AS sc2reset_maps_mods -FROM golang:1.23.3-alpine AS build_sc2_info_extractor +FROM golang:1.24.2-alpine AS build_sc2_info_extractor WORKDIR /app @@ -29,7 +29,8 @@ RUN mkdir -p replays/output COPY --from=build_sc2_info_extractor /app/SC2InfoExtractorGo /app/ # Copying all maps from from SC2ReSet to the maps directory for processing: # FIXME: Map files are not copied to the container: -COPY --from=sc2reset_maps_mods /sc2reset_maps_mods/maps/cn_maps /app/maps/ -COPY --from=sc2reset_maps_mods /sc2reset_maps_mods/maps/sc2reset_maps /app/maps/ +COPY --from=sc2reset_maps_mods /sc2reset_maps_mods/maps/cn_maps /app/dependencies/maps/ +COPY --from=sc2reset_maps_mods /sc2reset_maps_mods/maps/sc2reset_maps /app/dependencies/maps/ +COPY --from=sc2reset_maps_mods /sc2reset_maps_mods/other_dependencies /app/dependencies/other_dependencies/ ENTRYPOINT ["/app/SC2InfoExtractorGo"] diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 2e26ed0..cbe6bd9 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -4,7 +4,7 @@ FROM kaszanas/sc2replaytestdata:latest AS sc2replaytestdata # All of the maps and mods downloaded when processing SC2ReSet replays: FROM kaszanas/sc2reset_maps_mods:latest AS sc2reset_maps_mods -FROM golang:1.23.3 AS build_sc2_info_extractor +FROM golang:1.24.2 AS build_sc2_info_extractor # Install rsync for file copying: RUN apt-get update && apt-get install -y rsync @@ -15,8 +15,9 @@ COPY --from=sc2replaytestdata /sc2replaytestdata /test_files/test_replays/ WORKDIR /app # Copying all maps from from SC2ReSet to the maps directory for processing: -COPY --from=sc2reset_maps_mods /sc2reset_maps_mods/maps/cn_maps /app/maps/ -COPY --from=sc2reset_maps_mods /sc2reset_maps_mods/maps/sc2reset_maps /app/maps/ +COPY --from=sc2reset_maps_mods /sc2reset_maps_mods/maps/cn_maps /app/dependencies/maps/ +COPY --from=sc2reset_maps_mods /sc2reset_maps_mods/maps/sc2reset_maps /app/dependencies/maps/ +COPY --from=sc2reset_maps_mods /sc2reset_maps_mods/other_dependencies /app/dependencies/other_dependencies/ # Copy Golang dependency definitions: COPY go.mod go.sum /app/ From 50ca0da0c561c8b414cb56de54a8e5c3935674cb Mon Sep 17 00:00:00 2001 From: Kaszanas <34846245+Kaszanas@users.noreply.github.com> Date: Mon, 5 May 2025 01:51:56 +0200 Subject: [PATCH 2/9] Ran go mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index ea6892c..4241301 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,6 @@ github.com/icza/mpq v0.0.0-20230330132843-d3cdc0b651b7 h1:uWfnpztXMlK2068Uuv23eG github.com/icza/mpq v0.0.0-20230330132843-d3cdc0b651b7/go.mod h1:uZjJdSdZs2x2Gq+6/NdJE7nJ1upyftiNnw1ZMCFH+tc= github.com/icza/s2prot v1.5.2-0.20241207072335-d0e305d1c9c8 h1:dX42iKZ/pURGBBb/f2CXjwAseqP4UKre/jiUPFz+1UU= github.com/icza/s2prot v1.5.2-0.20241207072335-d0e305d1c9c8/go.mod h1:Aw3BgGOZ83Qkxmz90i22WFCMiDU4oOFT5D3hoDLBl3E= -github.com/icza/s2prot v1.5.2 h1:lc24kcR0FcUlHPB8l7qmJ4zeEJDKYgTacr/oCD66h+w= -github.com/icza/s2prot v1.5.2/go.mod h1:Aw3BgGOZ83Qkxmz90i22WFCMiDU4oOFT5D3hoDLBl3E= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= From e2e9792653029c4398c25db2f21c09daee2449e7 Mon Sep 17 00:00:00 2001 From: Kaszanas <34846245+Kaszanas@users.noreply.github.com> Date: Mon, 5 May 2025 01:56:07 +0200 Subject: [PATCH 3/9] First draft of downloading other dependencies --- dataproc/dataproc_pipeline_test.go | 6 +- dataproc/downloader/download_all_maps.go | 35 ++-- dataproc/downloader/downloader.go | 168 ++++++++++++------ .../downloader/map_downloader_pipeline.go | 53 ++++-- dataproc/sc2_map_processing/sc2_map_utils.go | 132 +++++++++----- main.go | 14 +- utils/flag_utils.go | 38 ++-- 7 files changed, 280 insertions(+), 166 deletions(-) diff --git a/dataproc/dataproc_pipeline_test.go b/dataproc/dataproc_pipeline_test.go index a4f39fe..acab43c 100644 --- a/dataproc/dataproc_pipeline_test.go +++ b/dataproc/dataproc_pipeline_test.go @@ -145,8 +145,8 @@ func testPipelineWrapperWithDir( flags := utils.CLIFlags{ InputDirectory: replayInputPath, OutputDirectory: thisTestOutputDir, - OnlyMapsDownload: false, - MapsDirectory: "../maps/", + OnlyDependencyDownload: false, + DependencyDirectory: "../dependencies/", NumberOfThreads: 1, NumberOfPackages: 1, PerformIntegrityCheck: true, @@ -166,7 +166,7 @@ func testPipelineWrapperWithDir( // Auxiliary files will be placed in the same directory as the log file: foreignToEnglishMappingFilepath := logFlags.LogPath + "map_foreign_to_english_mapping.json" - foreignToEnglishMapping := downloader.MapDownloaderPipeline( + foreignToEnglishMapping := downloader.DependencyDownloaderPipeline( sliceOfFiles, foreignToEnglishMappingFilepath, flags, diff --git a/dataproc/downloader/download_all_maps.go b/dataproc/downloader/download_all_maps.go index 719c217..902e26f 100644 --- a/dataproc/downloader/download_all_maps.go +++ b/dataproc/downloader/download_all_maps.go @@ -3,53 +3,54 @@ package downloader import ( "net/url" + "github.com/Kaszanas/SC2InfoExtractorGo/dataproc/sc2_map_processing" "github.com/Kaszanas/SC2InfoExtractorGo/utils" log "github.com/sirupsen/logrus" ) -// downloadAllSC2Maps download all of the maps from the replays +// DownloadAllSC2Dependencies download all of the dependencies from the replays // if the replays were not processed before. -func DownloadAllSC2Maps( +func DownloadAllSC2Dependencies( downloaderSharedState *DownloaderSharedState, - URLsToDownload map[url.URL]string, + URLsToDownload map[url.URL]sc2_map_processing.ReplayFilenameIsMap, cliFlags utils.CLIFlags, ) error { log.WithFields(log.Fields{ - "mapsDirectory": cliFlags.MapsDirectory}, - ).Debug("Entered downloadAllSC2Maps()") + "dependencyDirectory": cliFlags.DependencyDirectory}, + ).Debug("Entered DownloadAllSC2Dependencies()") defer downloaderSharedState.WorkerPool.StopAndWait() // Progress bar: - progressBarDownloadMaps := utils.NewProgressBar( + progressBarDownloadDependencies := utils.NewProgressBar( len(URLsToDownload), - "[2/4] Downloading maps: ", + "[2/4] Downloading dependencies: ", ) - defer progressBarDownloadMaps.Close() - for url, mapHashAndExtension := range URLsToDownload { + defer progressBarDownloadDependencies.Close() + for url, filenameAndIsMap := range URLsToDownload { // If the replay was not processed previosly, // open the replay, get map information, // download the map, and save it to the drive. - err := DownloadMapIfNotExists( + err := DownloadDependencyIfNotExists( downloaderSharedState, - mapHashAndExtension, + filenameAndIsMap, url, - progressBarDownloadMaps, + progressBarDownloadDependencies, ) if err != nil { log.WithFields(log.Fields{ - "mapURL": url.String(), - "mapHashAndExtension": mapHashAndExtension, + "mapURL": url.String(), + "dependencyHashAndExtension": filenameAndIsMap.DependencyFilename, }).Error("Failed to download the map.") } } // Wait Stop and wait without defer, - // all of the maps need to finish downloading before the processing starts: + // all of the dependencies need to finish downloading before the processing starts: downloaderSharedState.WorkerPool.StopAndWait() - progressBarDownloadMaps.Close() + progressBarDownloadDependencies.Close() - log.Debug("Finished downloadAllSC2Maps()") + log.Debug("Finished DownloadAllSC2Dependencies()") return nil } diff --git a/dataproc/downloader/downloader.go b/dataproc/downloader/downloader.go index 6d2dbca..43f1cf5 100644 --- a/dataproc/downloader/downloader.go +++ b/dataproc/downloader/downloader.go @@ -18,7 +18,7 @@ import ( ) // Mapping from hash and type to the name of the map -// Set of maps that are currently being downloaded, +// Set of dependencies that are currently being downloaded, // this avoids starting multiple downloads of the same map: // REVIEW: How to effectively wait for the download to finish // REVIEW: if another thread requests the same map name? @@ -27,14 +27,16 @@ import ( type DownloaderSharedState struct { // Directory where the maps will be downloaded: MapDownloadDirectory string // NOT_MODIFIABLE Directory where the maps are downloaded - // Set of maps that already exist on the drive: - DownloadedMapsSet *map[string]struct{} // MODIFIABLE Mapping from filename to english map name - // Map of maps that are currently being downloaded to the channels that are waiting for the download to finish: + // Directory where other dependencies will be downloaded: + DependencyDownloadDirectory string // NOT_MODIFIABLE Directory where other dependencies are downloaded + // Set of dependencies that already exist on the drive: + DownloadedDependenciesSet *map[string]struct{} // MODIFIABLE Mapping from filename to english map name + // Map of dependencies that are currently being downloaded to the channels that are waiting for the download to finish: CurrentlyDownloading *map[string][]chan DownloadTaskReturnChannelInfo // MODIFIABLE Mapping from filename to list of channels to be notified when download finishes // Mutex for shared state: SharedRWMutex *sync.RWMutex // MODIFIABLE Mutex for shared state - // Worker pool for downloading maps in parallel: - WorkerPool *pond.WorkerPool // Worker pool for downloading maps. + // Worker pool for downloading dependencies in parallel: + WorkerPool *pond.WorkerPool // Worker pool for downloading dependencies. } // Constructor for new downloader shared state @@ -43,8 +45,13 @@ func NewDownloaderSharedState( cliFlags utils.CLIFlags, ) (DownloaderSharedState, error) { + mapsDirectory := filepath.Join( + cliFlags.DependencyDirectory, + "maps", + ) + existingFilesMapsSet, err := file_utils.ExistingFilesSet( - cliFlags.MapsDirectory, ".s2ma", + mapsDirectory, ".s2ma", ) if err != nil { log.WithField("error", err). @@ -53,7 +60,7 @@ func NewDownloaderSharedState( } log.WithFields(log.Fields{ - "mapsDirectory": cliFlags.MapsDirectory, + "mapsDirectory": mapsDirectory, "existingFilesMapsSet": len(existingFilesMapsSet)}, ).Debug("Entered NewDownloaderSharedState()") @@ -81,23 +88,47 @@ func NewDownloaderSharedState( } } + otherDependenciesDirectory := filepath.Join( + cliFlags.DependencyDirectory, + "other_dependencies", + ) + + otherDependencyFilesSet, err := file_utils.ExistingFilesSet( + otherDependenciesDirectory, ".s2ma", + ) + if err != nil { + log.WithField("error", err). + Error("Failed to get existing other dependencies files set.") + return DownloaderSharedState{}, err + } + + // Combine the two sets of existing files: + dependenciesSet := make(map[string]struct{}) + for existingMapFilepath := range existingFilesMapsSet { + dependenciesSet[existingMapFilepath] = struct{}{} + } + for existingOtherDependencyFilepath := range otherDependencyFilesSet { + dependenciesSet[existingOtherDependencyFilepath] = struct{}{} + } + return DownloaderSharedState{ - MapDownloadDirectory: cliFlags.MapsDirectory, - DownloadedMapsSet: &existingFilesMapsSet, - CurrentlyDownloading: &map[string][]chan DownloadTaskReturnChannelInfo{}, - SharedRWMutex: &sync.RWMutex{}, - WorkerPool: pond.New(4, cliFlags.NumberOfThreads*2, pond.Strategy(pond.Eager())), + MapDownloadDirectory: mapsDirectory, + DependencyDownloadDirectory: otherDependenciesDirectory, + DownloadedDependenciesSet: &dependenciesSet, + CurrentlyDownloading: &map[string][]chan DownloadTaskReturnChannelInfo{}, + SharedRWMutex: &sync.RWMutex{}, + WorkerPool: pond.New(4, cliFlags.NumberOfThreads*2, pond.Strategy(pond.Eager())), }, nil } // DownloadTaskState holds all of the information needed for the download task. type DownloadTaskState struct { - mapDownloadDirectory string - mapHashAndExtension string - mapURL url.URL - downloadedMapsSet *map[string]struct{} - currentlyDownloading *map[string][]chan DownloadTaskReturnChannelInfo - sharedRWMutex *sync.RWMutex + dependencyDownloadDirectory string + dependencyFilenameIsMap sc2_map_processing.ReplayFilenameIsMap + dependencyURL url.URL + downloadedDependenciesSet *map[string]struct{} + currentlyDownloading *map[string][]chan DownloadTaskReturnChannelInfo + sharedRWMutex *sync.RWMutex } // DownloadTaskReturnChannelInfo holds the information needed to return @@ -107,9 +138,9 @@ type DownloadTaskReturnChannelInfo struct { } // TODO: Change name: -func DownloadMapIfNotExists( +func DownloadDependencyIfNotExists( downloaderSharedState *DownloaderSharedState, - mapHashAndExtension string, + filenameAndIsMap sc2_map_processing.ReplayFilenameIsMap, mapURL url.URL, progressBar *progressbar.ProgressBar, ) error { @@ -117,34 +148,34 @@ func DownloadMapIfNotExists( defer func() { if err := progressBar.Add(1); err != nil { log.WithField("error", err). - Error("Error updating progress bar in DownloadMapIfNotExists") + Error("Error updating progress bar in DownloadDependencyIfNotExists") } }() log.WithFields( log.Fields{ - "mapHashAndExtension": mapHashAndExtension, - "mapURL": mapURL.String(), + "dependencyFilename": filenameAndIsMap.DependencyFilename, + "mapURL": mapURL.String(), }, - ).Debug("Entered getEnglishMapNameDownloadIfNotExists()") + ).Debug("Entered DownloadDependencyIfNotExists()") downloadTaskInfoChannel := dispatchMapDownloadTask( *downloaderSharedState, - mapHashAndExtension, + filenameAndIsMap, mapURL, ) if downloadTaskInfoChannel == nil { return nil } - // Wait for channel to finish downloading the map. + // Wait for channel to finish downloading the dependency. taskStatus := <-downloadTaskInfoChannel if taskStatus.err != nil { - log.WithField("error", taskStatus.err).Error("Error downloading map") - return fmt.Errorf("error downloading map: %v", taskStatus.err) + log.WithField("error", taskStatus.err).Error("Error downloading dependency") + return fmt.Errorf("error downloading dependency: %v", taskStatus.err) } - log.Debug("Finished getEnglishMapNameDownloadIfNotExists()") + log.Debug("Finished DownloadDependencyIfNotExists()") return nil } @@ -152,7 +183,7 @@ func DownloadMapIfNotExists( // the map is not available within the shared state under the mapHashAndExtensionToName. func dispatchMapDownloadTask( downloaderSharedState DownloaderSharedState, - mapHashAndExtension string, + filenameAndIsMap sc2_map_processing.ReplayFilenameIsMap, mapURL url.URL, ) chan DownloadTaskReturnChannelInfo { @@ -162,14 +193,25 @@ func dispatchMapDownloadTask( // REVIEW: Is this the best way to go about it? // This is required because downloaded maps set contains full paths to the maps: - maybeMapFilepath := filepath.Join( - downloaderSharedState.MapDownloadDirectory, - mapHashAndExtension, - ) + + maybeDependencyFilepath := "" + switch filenameAndIsMap.IsMap { + case true: + maybeDependencyFilepath = filepath.Join( + downloaderSharedState.MapDownloadDirectory, + filenameAndIsMap.DependencyFilename, + ) + case false: + maybeDependencyFilepath = filepath.Join( + downloaderSharedState.DependencyDownloadDirectory, + filenameAndIsMap.DependencyFilename, + ) + } + // Check if the english map name was already read from the drive, return if present: - _, ok := (*downloaderSharedState.DownloadedMapsSet)[maybeMapFilepath] + _, ok := (*downloaderSharedState.DownloadedDependenciesSet)[maybeDependencyFilepath] if ok { - log.WithField("mapHashAndExtension", mapHashAndExtension). + log.WithField("mapHashAndExtension", filenameAndIsMap.DependencyFilename). Info("Map name was already processed in mapHashAndExtensionToName, returning.") return nil } @@ -178,35 +220,43 @@ func dispatchMapDownloadTask( downloadTaskInfoChannel := make(chan DownloadTaskReturnChannelInfo) // Check if key is in currently downloading: - listOfChannels, ok := (*downloaderSharedState.CurrentlyDownloading)[mapHashAndExtension] + listOfChannels, ok := (*downloaderSharedState.CurrentlyDownloading)[filenameAndIsMap.DependencyFilename] if ok { // If it is downloading then add the channel to the list of channels waiting for result // Map is being downloaded, add it to the list of currently downloading maps: - log.WithField("mapHashAndExtension", mapHashAndExtension). - Info("Map is being downloaded, adding channel to receive the result.") - (*downloaderSharedState.CurrentlyDownloading)[mapHashAndExtension] = + log.WithField("mapHashAndExtension", filenameAndIsMap). + Info("Dependency is being downloaded, adding channel to receive the result.") + (*downloaderSharedState.CurrentlyDownloading)[filenameAndIsMap.DependencyFilename] = append(listOfChannels, downloadTaskInfoChannel) } else { - log.WithField("mapHashAndExtension", mapHashAndExtension). - Info("Map is not being downloaded, adding to download queue.") + log.WithField("mapHashAndExtension", filenameAndIsMap). + Info("Dependency is not being downloaded, adding to download queue.") + taskState := DownloadTaskState{ - mapDownloadDirectory: downloaderSharedState.MapDownloadDirectory, - downloadedMapsSet: downloaderSharedState.DownloadedMapsSet, - currentlyDownloading: downloaderSharedState.CurrentlyDownloading, - mapHashAndExtension: mapHashAndExtension, - mapURL: mapURL, - sharedRWMutex: downloaderSharedState.SharedRWMutex, + downloadedDependenciesSet: downloaderSharedState.DownloadedDependenciesSet, + currentlyDownloading: downloaderSharedState.CurrentlyDownloading, + dependencyFilenameIsMap: filenameAndIsMap, + dependencyURL: mapURL, + sharedRWMutex: downloaderSharedState.SharedRWMutex, } + + switch filenameAndIsMap.IsMap { + case true: + taskState.dependencyDownloadDirectory = downloaderSharedState.MapDownloadDirectory + case false: + taskState.dependencyDownloadDirectory = downloaderSharedState.DependencyDownloadDirectory + } + // if it is not then add key to the map and create one element // slice with the channel and submit the download task to the worker pool: - (*downloaderSharedState.CurrentlyDownloading)[mapHashAndExtension] = + (*downloaderSharedState.CurrentlyDownloading)[filenameAndIsMap.DependencyFilename] = []chan DownloadTaskReturnChannelInfo{downloadTaskInfoChannel} downloaderSharedState.WorkerPool.Submit( func() { // Errors are written to directly to the channel, // each of requesting goroutines will receive the error from // this function via the channel. - downloadSingleMap(taskState) + downloadSingleDependency(taskState) }, ) } @@ -215,17 +265,17 @@ func dispatchMapDownloadTask( return downloadTaskInfoChannel } -// downloadSingleMap handles downloading a single map based on an URL passed through +// downloadSingleDependency handles downloading a single map based on an URL passed through // the task state. -func downloadSingleMap(taskState DownloadTaskState) { +func downloadSingleDependency(taskState DownloadTaskState) { log.Debug("Entered downloadSingleMap()") outputFilepath := filepath.Join( - taskState.mapDownloadDirectory, - taskState.mapHashAndExtension, + taskState.dependencyDownloadDirectory, + taskState.dependencyFilenameIsMap.DependencyFilename, ) - response, err := http.Get(taskState.mapURL.String()) + response, err := http.Get(taskState.dependencyURL.String()) if err != nil { sendDownloadTaskReturnInfoToChannels( &taskState, @@ -282,11 +332,11 @@ func sendDownloadTaskReturnInfoToChannels( taskState.sharedRWMutex.Lock() defer taskState.sharedRWMutex.Unlock() - (*taskState.downloadedMapsSet)[taskState.mapHashAndExtension] = struct{}{} - for _, channel := range (*taskState.currentlyDownloading)[taskState.mapHashAndExtension] { + (*taskState.downloadedDependenciesSet)[taskState.dependencyFilenameIsMap.DependencyFilename] = struct{}{} + for _, channel := range (*taskState.currentlyDownloading)[taskState.dependencyFilenameIsMap.DependencyFilename] { channel <- DownloadTaskReturnChannelInfo{ err: err, } } - delete(*taskState.currentlyDownloading, taskState.mapHashAndExtension) + delete(*taskState.currentlyDownloading, taskState.dependencyFilenameIsMap.DependencyFilename) } diff --git a/dataproc/downloader/map_downloader_pipeline.go b/dataproc/downloader/map_downloader_pipeline.go index 42b3d4c..2e4456b 100644 --- a/dataproc/downloader/map_downloader_pipeline.go +++ b/dataproc/downloader/map_downloader_pipeline.go @@ -2,6 +2,7 @@ package downloader import ( "net/url" + "path/filepath" "github.com/Kaszanas/SC2InfoExtractorGo/dataproc/sc2_map_processing" "github.com/Kaszanas/SC2InfoExtractorGo/utils" @@ -9,24 +10,46 @@ import ( log "github.com/sirupsen/logrus" ) -func MapDownloaderPipeline( +func DependencyDownloaderPipeline( files []string, foreignToEnglishMappingFilepath string, cliFlags utils.CLIFlags, ) map[string]string { - if !cliFlags.SkipMapsDownload { - // Create maps directory if it doesn't exist: - err := file_utils.GetOrCreateDirectory(cliFlags.MapsDirectory) + if !cliFlags.SkipDependencyDownload { + // Create dependency directory if it doesn't exist: + err := file_utils.GetOrCreateDirectory(cliFlags.DependencyDirectory) if err != nil { - log.WithField("error", err).Error("Failed to create maps directory.") + log.WithField("error", err).Error("Failed to create dependencies directory.") + return nil + } + + // create the directory for map downloads if it doesn't exist: + mapsDirectory := filepath.Join( + cliFlags.DependencyDirectory, + "maps", + ) + err = file_utils.GetOrCreateDirectory(mapsDirectory) + if err != nil { + log.WithField("error", err).Error("Failed to create other maps directory.") + return nil + } + + // Create directory for other dependency downloads if it doesn't exist: + otherDependenciesDirectory := filepath.Join( + cliFlags.DependencyDirectory, + "other_dependencies", + ) + err = file_utils.GetOrCreateDirectory(otherDependenciesDirectory) + if err != nil { + log.WithField("error", err).Error("Failed to create other dependencies directory.") return nil } // REVIEW: Start Review: // STAGE ONE PRE-PROCESS: // Get all map URLs into a set: - URLsToDownload, err := getURLsForMissingMaps( + URLsToDownload, err := getURLsForMissingDependencies( files, cliFlags, ) @@ -41,7 +64,7 @@ func MapDownloaderPipeline( // STAGE-TWO PRE-PROCESS: Attempt downloading all SC2 maps from the read replays. // Download all SC2 maps from the replays if they were not processed before: - downloadMissingMaps(URLsToDownload, cliFlags) + downloadMissingDependencies(URLsToDownload, cliFlags) } // STAGE-Three PRE-PROCESS: @@ -56,13 +79,13 @@ func MapDownloaderPipeline( return mainForeignToEnglishMapping } -func getURLsForMissingMaps( +func getURLsForMissingDependencies( files []string, cliFlags utils.CLIFlags, -) (map[url.URL]string, error) { +) (map[url.URL]sc2_map_processing.ReplayFilenameIsMap, error) { existingMapFilesSet, err := file_utils.ExistingFilesSet( - cliFlags.MapsDirectory, ".s2ma", + cliFlags.DependencyDirectory, ".s2ma", ) if err != nil { log.WithField("error", err). @@ -71,7 +94,7 @@ func getURLsForMissingMaps( } URLsToDownload, err := sc2_map_processing. - GetAllReplaysMapURLs( + GetAllReplaysDependencyURLs( files, existingMapFilesSet, cliFlags, @@ -90,8 +113,8 @@ type URLToFileTuple struct { Filename string } -func downloadMissingMaps( - URLsToDownload map[url.URL]string, +func downloadMissingDependencies( + URLsToDownload map[url.URL]sc2_map_processing.ReplayFilenameIsMap, cliFlags utils.CLIFlags, ) { @@ -105,7 +128,7 @@ func downloadMissingMaps( return } - err = DownloadAllSC2Maps( + err = DownloadAllSC2Dependencies( &downloaderSharedState, URLsToDownload, cliFlags, @@ -122,7 +145,7 @@ func readMapNamesFromMapFiles( ) map[string]string { existingMapFilesSet, err := file_utils.ExistingFilesSet( - cliFlags.MapsDirectory, ".s2ma", + cliFlags.DependencyDirectory, ".s2ma", ) if err != nil { log.WithField("error", err). diff --git a/dataproc/sc2_map_processing/sc2_map_utils.go b/dataproc/sc2_map_processing/sc2_map_utils.go index 8e3035d..8a9258f 100644 --- a/dataproc/sc2_map_processing/sc2_map_utils.go +++ b/dataproc/sc2_map_processing/sc2_map_utils.go @@ -20,27 +20,33 @@ import ( // ReplayProcessingChannelContents is a struct that is used to pass data // between the orchestrator and the workers in the pipeline. -type ReplayMapExtractProcessingChannel struct { +type ReplayDependencyExtractProcessingChannel struct { Index int ChunkOfFiles []string } -type ReplayProcessingMapInfo struct { - ReplayFilename string - MapHashAndExtension string +type ReplayProcessingDependencyInfo struct { + ReplayFilename string + DependencyHashAndExtension string + IsMap bool +} + +type ReplayFilenameIsMap struct { + DependencyFilename string + IsMap bool } type ExtractMapChannelContents struct { - mapOfURLs map[url.URL]ReplayProcessingMapInfo + mapOfURLs map[url.URL]ReplayProcessingDependencyInfo } -// GetAllReplaysMapURLs retrieves the map URLs from the replay files. -func GetAllReplaysMapURLs( +// GetAllReplaysDependencyURLs retrieves the map URLs from the replay files. +func GetAllReplaysDependencyURLs( files []string, - mapsOnDriveSet map[string]struct{}, + dependenciesOnDriveSet map[string]struct{}, cliFlags utils.CLIFlags, ) ( - map[url.URL]string, + map[url.URL]ReplayFilenameIsMap, error, ) { @@ -56,11 +62,11 @@ func GetAllReplaysMapURLs( // If it is specified by the user to perform the processing without // multiprocessing GOMAXPROCS needs to be set to 1 in order to allow 1 thread: runtime.GOMAXPROCS(cliFlags.NumberOfThreads) - inputChannel := make(chan ReplayMapExtractProcessingChannel, cliFlags.NumberOfThreads+1) + inputChannel := make(chan ReplayDependencyExtractProcessingChannel, cliFlags.NumberOfThreads+1) outputChannel := make(chan ExtractMapChannelContents, cliFlags.NumberOfThreads+1) // Creating chunks of files for data parallel multiprocessing: - downloadMapsChunks, _ := chunk_utils.GetChunkListAndPackageBool( + downloadDependenciesChunks, _ := chunk_utils.GetChunkListAndPackageBool( files, 0, cliFlags.NumberOfThreads, @@ -73,7 +79,7 @@ func GetAllReplaysMapURLs( wg.Add(cliFlags.NumberOfThreads) // Spin up workers waiting for chunks to process: - for i := 0; i < cliFlags.NumberOfThreads; i++ { + for range cliFlags.NumberOfThreads { go createMapExtractingGoroutines( inputChannel, outputChannel, @@ -88,8 +94,8 @@ func GetAllReplaysMapURLs( // the information about which maps should be downloaded will be put into a single // map which will handle the duplicates: // Passing the chunks to input channel before the workers start processing: - for index, chunk := range downloadMapsChunks { - inputChannel <- ReplayMapExtractProcessingChannel{ + for index, chunk := range downloadDependenciesChunks { + inputChannel <- ReplayDependencyExtractProcessingChannel{ Index: index, ChunkOfFiles: chunk, } @@ -107,15 +113,18 @@ func GetAllReplaysMapURLs( // Consume the output from the workers. This is needed to get rid of the // duplicate map URLs, multiple replays can have the same map: - toDownloadURLToFileMap := make(map[url.URL]string) + toDownloadDependencyToFileMap := make(map[url.URL]ReplayFilenameIsMap) for output := range outputChannel { - for url := range output.mapOfURLs { - replayHashExtension := output.mapOfURLs[url].MapHashAndExtension + for replayDependencyURL := range output.mapOfURLs { + + replayProcessingDependencyInfo := output.mapOfURLs[replayDependencyURL] + + replayHashExtension := replayProcessingDependencyInfo.DependencyHashAndExtension // The map will have to be downloaded only if it is not already existing // on the disk: - _, ok := mapsOnDriveSet[replayHashExtension] + _, ok := dependenciesOnDriveSet[replayHashExtension] if ok { // the map is already downloaded, skip it: log.WithField("map", replayHashExtension). @@ -123,21 +132,24 @@ func GetAllReplaysMapURLs( continue } - toDownloadURLToFileMap[url] = replayHashExtension + toDownloadDependencyToFileMap[replayDependencyURL] = ReplayFilenameIsMap{ + DependencyFilename: replayHashExtension, + IsMap: replayProcessingDependencyInfo.IsMap, + } } } - log.WithField("nMapsToDownload", len(toDownloadURLToFileMap)). + log.WithField("nMapsToDownload", len(toDownloadDependencyToFileMap)). Debug("Finished GetAllReplaysMapURLs()") // Return all of the URLs - return toDownloadURLToFileMap, nil + return toDownloadDependencyToFileMap, nil } // createMapExtractingGoroutines creates the goroutines that process the replay files // and extract the map URLs. func createMapExtractingGoroutines( - inputChannel chan ReplayMapExtractProcessingChannel, + inputChannel chan ReplayDependencyExtractProcessingChannel, outputChannel chan ExtractMapChannelContents, progressBar *progressbar.ProgressBar, wg *sync.WaitGroup, @@ -151,7 +163,7 @@ func createMapExtractingGoroutines( return } // Process the chunk of files and add the URLs to the map - mapOfURLs := make(map[url.URL]ReplayProcessingMapInfo) + mapOfURLs := make(map[url.URL]ReplayProcessingDependencyInfo) for _, replayFullFilepath := range channelContents.ChunkOfFiles { // Filling out the map of urls that will be returned through the output channel // the caller will handle the consumption of the output channel and @@ -181,7 +193,7 @@ func createMapExtractingGoroutines( func processFileExtractMapURL( progressBar *progressbar.ProgressBar, replayFullFilepath string, - urls map[url.URL]ReplayProcessingMapInfo, + urls map[url.URL]ReplayProcessingDependencyInfo, ) bool { // Lambda to process the replay file to have @@ -196,7 +208,7 @@ func processFileExtractMapURL( }() replayFilename := filepath.Base(replayFullFilepath) - mapURL, mapHashAndExtension, err := getURL(replayFullFilepath) + dependencyURLs, err := getURL(replayFullFilepath) if err != nil { log.WithFields(log.Fields{ "error": err, @@ -205,42 +217,52 @@ func processFileExtractMapURL( return false } - // Store the map URL and hash: - urls[mapURL] = ReplayProcessingMapInfo{ - ReplayFilename: replayFilename, - MapHashAndExtension: mapHashAndExtension, + // Store the dependency URLs, hash and information if the dependency is a map: + for _, dependencyInfo := range dependencyURLs { + urls[*dependencyInfo.URL] = ReplayProcessingDependencyInfo{ + ReplayFilename: replayFilename, + DependencyHashAndExtension: dependencyInfo.HashAndExtensionMerged, + IsMap: dependencyInfo.IsMap, + } } return true } // getURL retrieves the map URL from the replay file. -func getURL(replayFullFilepath string) (url.URL, string, error) { +func getURL(replayFullFilepath string) ([]SC2DependencyInformation, error) { // Assume getURLsFromReplay is a function that // returns a slice of URLs from a replay file replayData, err := rep.NewFromFile(replayFullFilepath) if err != nil { log.WithFields(log.Fields{"file": replayFullFilepath, "error": err}). Error("Failed to read replay file to retrieve map data") - return url.URL{}, "", err + return []SC2DependencyInformation{}, err } - mapURL, mapHashAndExtension, mapRetrieved := - GetMapURLAndHashFromReplayData(replayData) + dependencyURLs, mapRetrieved := + GetDependencyURLsAndHashFromReplayData(replayData) if !mapRetrieved { log.WithField("file", replayFullFilepath). Warning("Failed to get map URL and hash from replay data") - return url.URL{}, "", fmt.Errorf("failed to get map URL and hash from replay data") + return []SC2DependencyInformation{}, fmt.Errorf("failed to get map URL and hash from replay data") } - return mapURL, mapHashAndExtension, nil + return dependencyURLs, nil +} + +type SC2DependencyInformation struct { + URL *url.URL + URLString string + HashAndExtensionMerged string + IsMap bool } -// GetMapURLAndHashFromReplayData extracts the map URL, +// GetDependencyURLsAndHashFromReplayData extracts the map URL, // hash, and file extension from the replay data. -func GetMapURLAndHashFromReplayData( +func GetDependencyURLsAndHashFromReplayData( replayData *rep.Rep, -) (url.URL, string, bool) { +) ([]SC2DependencyInformation, bool) { log.Debug("Entered getMapURLAndHashFromReplayData()") cacheHandles := replayData.Details.CacheHandles() @@ -255,7 +277,7 @@ func GetMapURLAndHashFromReplayData( Warning( "Detected unsupported region! Won't download the map! Replay may fail further processing!", ) - return url.URL{}, "", false + return []SC2DependencyInformation{}, false } } @@ -268,14 +290,32 @@ func GetMapURLAndHashFromReplayData( depotURL := region.DepotURL - hashAndExtensionMerged := fmt.Sprintf( - "%s.%s", - mapCacheHandle.Digest, - mapCacheHandle.Type, - ) - mapURL := depotURL.JoinPath(hashAndExtensionMerged) + dependencyURLs := []SC2DependencyInformation{} + + for index, dependency := range cacheHandles { + hashAndExtensionMerged := fmt.Sprintf( + "%s.%s", + dependency.Digest, + dependency.Type, + ) + dependencyURL := depotURL.JoinPath(hashAndExtensionMerged) + + isMap := false + if index == len(cacheHandles)-1 { + isMap = true + } + + urlStringTuple := SC2DependencyInformation{ + URL: dependencyURL, + URLString: dependencyURL.String(), + HashAndExtensionMerged: hashAndExtensionMerged, + IsMap: isMap, + } + dependencyURLs = append(dependencyURLs, urlStringTuple) + } + log.Debug("Finished getMapURLAndHashFromReplayData()") - return *mapURL, hashAndExtensionMerged, true + return dependencyURLs, true } // ReadLocalizedDataFromMapGetForeignToEnglishMapping opens the map file (MPQ), diff --git a/main.go b/main.go index 8e99c15..b505edb 100644 --- a/main.go +++ b/main.go @@ -44,9 +44,9 @@ func mainReturnWithCode() int { log.WithFields(log.Fields{ "CLIflags.InputDirectory": CLIflags.InputDirectory, "CLIflags.OutputDirectory": CLIflags.OutputDirectory, - "CLIflags.OnlyMapsDownload": CLIflags.OnlyMapsDownload, - "CLIflags.SkipMapsDownload": CLIflags.SkipMapsDownload, - "CLIflags.MapsDirectory": CLIflags.MapsDirectory, + "CLIflags.OnlyDependencyDownload": CLIflags.OnlyDependencyDownload, + "CLIflags.SkipDependencyDownload": CLIflags.SkipDependencyDownload, + "CLIflags.DependencyDirectory": CLIflags.DependencyDirectory, "CLIflags.NumberOfPackages": CLIflags.NumberOfPackages, "CLIflags.PerformIntegrityCheck": CLIflags.PerformIntegrityCheck, "CLIflags.PerformValidityCheck": CLIflags.PerformValidityCheck, @@ -91,14 +91,14 @@ func mainReturnWithCode() int { return 1 } - // Downloading the maps for the files: - foreignToEnglishMapping := downloader.MapDownloaderPipeline( + // Downloading the dependencies for the files: + foreignToEnglishMapping := downloader.DependencyDownloaderPipeline( listOfInputFiles, foreignToEnglishMappingFilepath, CLIflags, ) - if CLIflags.OnlyMapsDownload { - log.Info("Only maps download was chosen. Exiting.") + if CLIflags.OnlyDependencyDownload { + log.Info("Only dependency download was chosen. Exiting.") return 0 } diff --git a/utils/flag_utils.go b/utils/flag_utils.go index 6714018..a2fd59e 100644 --- a/utils/flag_utils.go +++ b/utils/flag_utils.go @@ -20,9 +20,9 @@ type LogFlags struct { type CLIFlags struct { InputDirectory string OutputDirectory string - OnlyMapsDownload bool - SkipMapsDownload bool - MapsDirectory string + OnlyDependencyDownload bool + SkipDependencyDownload bool + DependencyDirectory string NumberOfThreads int NumberOfPackages int PerformIntegrityCheck bool @@ -50,22 +50,22 @@ func ParseFlags() (CLIFlags, bool) { "Output directory where compressed zip packages will be saved.", ) - onlyMapDownload := flag.Bool( - "only_map_download", + onlyDependencyDownload := flag.Bool( + "only_dependency_download", false, `Flag specifying if the tool is supposed to only download - the maps and not process the replays.`, + the replay dependencies and not process the replays.`, ) - skipMapDownload := flag.Bool( - "skip_map_download", + skipDependencyDownload := flag.Bool( + "skip_dependency_download", false, - `Flag specifying if the tool is supposed to skip the map download`, + `Flag specifying if the tool is supposed to skip the dependency download.`, ) - mapsDirectory := flag.String( - "maps_directory", - "./maps/", - "Directory where the maps will be downloaded as a result of the replay processing.", + dependencyDirectory := flag.String( + "dependency_directory", + "./dependencies/", + "Directory where the replay dependencies will be downloaded as a result of the replay processing.", ) numberOfPackagesFlag := flag.Int( @@ -178,10 +178,10 @@ func ParseFlags() (CLIFlags, bool) { return CLIFlags{}, false } - absolutePathMapsDirectory, err := filepath.Abs(*mapsDirectory) + absolutePathDependencyDirectory, err := filepath.Abs(*dependencyDirectory) if err != nil { - log.WithField("mapsDirectory", *mapsDirectory). - Error("Failed to get the absolute path to the maps directory!") + log.WithField("dependencyDirectory", *dependencyDirectory). + Error("Failed to get the absolute path to the dependency directory!") return CLIFlags{}, false } @@ -193,9 +193,9 @@ func ParseFlags() (CLIFlags, bool) { flags := CLIFlags{ InputDirectory: absoluteInputDirectory, OutputDirectory: absolutePathOutputDirectory, - OnlyMapsDownload: *onlyMapDownload, - SkipMapsDownload: *skipMapDownload, - MapsDirectory: absolutePathMapsDirectory, + OnlyDependencyDownload: *onlyDependencyDownload, + SkipDependencyDownload: *skipDependencyDownload, + DependencyDirectory: absolutePathDependencyDirectory, NumberOfPackages: *numberOfPackagesFlag, PerformIntegrityCheck: *performIntegrityCheckFlag, PerformValidityCheck: *performValidityCheckFlag, From cd0a0b5163f2523f48d8268f6abf17087fb968c0 Mon Sep 17 00:00:00 2001 From: Kaszanas <34846245+Kaszanas@users.noreply.github.com> Date: Mon, 5 May 2025 01:56:34 +0200 Subject: [PATCH 4/9] Added new dependencies directory to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9c5e900..b3b772b 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ package_summary* test_replays_output/ *.s2ma maps/ +dependencies/ processed_replays.json downloaded_maps_for_replays.json map_foreign_to_english_mapping.json From 73ad2da74fbca11d255a51b034b3591dc182c86d Mon Sep 17 00:00:00 2001 From: Kaszanas <34846245+Kaszanas@users.noreply.github.com> Date: Mon, 5 May 2025 01:58:44 +0200 Subject: [PATCH 5/9] Adjusted README with dependency download capabilities --- README.md | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0ea8226..5f3b47b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ A tool meant to allow for quick data extraction from StarCraft 2 replay files ". > This steps assumes that you have Docker installed on your machine and working. > Additionally, you will either need to pull the docker image from DockerHub or build it locally as described in [Build Docker Images](CONTRIBUTING.md#build-docker-images). -The easiest way to run this tool is to use the provided Docker image release. Please note that the `maps` directory can be mounted to see what map files are downloaded, and what map files were pre-seeded with the Docker Image. Other command line options are described in [CLI Options](#cli-options). +The easiest way to run this tool is to use the provided Docker image release. Please note that the `dependencies` directory can be mounted to see what dependency files are downloaded, and what dependency files were pre-seeded with the Docker Image. Other command line options are described in [CLI Options](#cli-options). Commands below showcases the exampe usage of the tool. **Steps:** @@ -33,14 +33,14 @@ docker pull kaszanas/sc2infoextractorgo:latest -v :/app/replays/input \ -v :/app/replays/output \ -v :/app/logs \ - -v :/app/maps \ + -v :/app/dependencies \ sc2infoextractorgo:latest -help ``` 4. When you execute the processing command, please note that you need to pass the appropriate paths to the tool that are withing the container. The default paths are: - Input directory: `/app/replays/input` - Output directory: `/app/replays/output` - Logs directory: `/app/logs` - - Maps directory: `/app/maps` + - Dependency directory: `/app/dependencies` - Additionally the tool outputs some auxiliary files that can be used for debugging purposes. These are created in the main directory of the container `/app/`. @@ -74,11 +74,12 @@ SC2InfoExtractorGo.exe -help The following flags are available: ``` + -dependency_directory string + Directory where the replay dependencies will be downloaded as a result of the replay processing. (default "./dependencies/") + -game_mode_filter int + Specifies which game mode should be included from the processed files in a format of a binary flag: AllGameModes: 0b11111111 (default 0b11111111) (default 255) -help Show command usage - -game_mode_filter int - Specifies which game mode should be included from the processed files - in a format of a binary flag: AllGameModes: 0b11111111 (default 0b11111111) (default 255) -input string Input directory where .SC2Replay files are held. (default "./replays/input") -log_dir string @@ -89,18 +90,16 @@ The following flags are available: Error - 3, Warn - 4, Info - 5, Debug - 6, Trace - 7 (default 4) - -maps_directory string - Directory where the maps will be downloaded as a result of the replay processing. (default "./maps/") -max_procs int Specifies the number of logic cores of a processor that will be used for processing (default runtime.NumCPU()). (default 24) -number_of_packages int Provide a number of zip packages to be created and compressed into a zip archive. Please remember that this number needs to be lower - than the number of processed files. If set to 0, will omit the + than the number of processed files. If set to 0, will ommit the zip packaging and output .json directly to drive. (default 1) - -only_map_download + -only_dependency_download Flag specifying if the tool is supposed to only download - the maps and not process the replays. + the replay dependencies and not process the replays. -output string Output directory where compressed zip packages will be saved. (default "./replays/output") -perform_chat_anonymization @@ -122,8 +121,8 @@ The following flags are available: -perform_validity_checks Flag, specifying if the tool is supposed to use hardcoded validity checks and verify if the replay file variables are within 'common sense' ranges. - -skip_map_download - Flag specifying if the tool is supposed to skip the map download + -skip_dependency_download + Flag specifying if the tool is supposed to skip the dependency download. -with_cpu_profiler string Set path to the file where pprof cpu profiler will save its information. If this is empty no profiling is performed. @@ -142,7 +141,7 @@ This is required because of the multiprocessing nature of our code that needs to ### Map Translation Support -Existing implementation downloads the maps from the Blizzard servers. This is to normalize the map names to English language. When there is no internet connection available, our tool should fallback to reading the map names from the files placed in the ```./maps``` directory. +Existing implementation downloads the maps from the Blizzard servers. This is to normalize the map names to English language. When there is no internet connection available, our tool should fallback to reading the map names from the files placed in the ```./dependencies``` directory. ### Filtering Capabilities From 6d4f29f95899dea0d8bef45662eddaddf7970fa7 Mon Sep 17 00:00:00 2001 From: Kaszanas <34846245+Kaszanas@users.noreply.github.com> Date: Mon, 5 May 2025 02:05:24 +0200 Subject: [PATCH 6/9] Adjusted logs, and old naming --- dataproc/downloader/downloader.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dataproc/downloader/downloader.go b/dataproc/downloader/downloader.go index 43f1cf5..04492b5 100644 --- a/dataproc/downloader/downloader.go +++ b/dataproc/downloader/downloader.go @@ -224,12 +224,12 @@ func dispatchMapDownloadTask( if ok { // If it is downloading then add the channel to the list of channels waiting for result // Map is being downloaded, add it to the list of currently downloading maps: - log.WithField("mapHashAndExtension", filenameAndIsMap). + log.WithField("DependencyFilename", filenameAndIsMap.DependencyFilename). Info("Dependency is being downloaded, adding channel to receive the result.") (*downloaderSharedState.CurrentlyDownloading)[filenameAndIsMap.DependencyFilename] = append(listOfChannels, downloadTaskInfoChannel) } else { - log.WithField("mapHashAndExtension", filenameAndIsMap). + log.WithField("DependencyFilename", filenameAndIsMap.DependencyFilename). Info("Dependency is not being downloaded, adding to download queue.") taskState := DownloadTaskState{ @@ -268,7 +268,7 @@ func dispatchMapDownloadTask( // downloadSingleDependency handles downloading a single map based on an URL passed through // the task state. func downloadSingleDependency(taskState DownloadTaskState) { - log.Debug("Entered downloadSingleMap()") + log.Debug("Entered downloadSingleDependency()") outputFilepath := filepath.Join( taskState.dependencyDownloadDirectory, From ab8f75950ca744e186252e8dfbf4df3298d2bbbf Mon Sep 17 00:00:00 2001 From: Kaszanas <34846245+Kaszanas@users.noreply.github.com> Date: Mon, 5 May 2025 02:11:49 +0200 Subject: [PATCH 7/9] Further improvement old variable names in logging --- dataproc/downloader/downloader.go | 4 ++-- dataproc/sc2_map_processing/sc2_map_utils.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dataproc/downloader/downloader.go b/dataproc/downloader/downloader.go index 04492b5..7695c52 100644 --- a/dataproc/downloader/downloader.go +++ b/dataproc/downloader/downloader.go @@ -211,8 +211,8 @@ func dispatchMapDownloadTask( // Check if the english map name was already read from the drive, return if present: _, ok := (*downloaderSharedState.DownloadedDependenciesSet)[maybeDependencyFilepath] if ok { - log.WithField("mapHashAndExtension", filenameAndIsMap.DependencyFilename). - Info("Map name was already processed in mapHashAndExtensionToName, returning.") + log.WithField("DependencyFilename", filenameAndIsMap.DependencyFilename). + Info("Dependency name was already processed in DownloadedDependenciesSet, returning.") return nil } diff --git a/dataproc/sc2_map_processing/sc2_map_utils.go b/dataproc/sc2_map_processing/sc2_map_utils.go index 8a9258f..d2f2039 100644 --- a/dataproc/sc2_map_processing/sc2_map_utils.go +++ b/dataproc/sc2_map_processing/sc2_map_utils.go @@ -120,20 +120,20 @@ func GetAllReplaysDependencyURLs( replayProcessingDependencyInfo := output.mapOfURLs[replayDependencyURL] - replayHashExtension := replayProcessingDependencyInfo.DependencyHashAndExtension + dependencyHashExtension := replayProcessingDependencyInfo.DependencyHashAndExtension // The map will have to be downloaded only if it is not already existing // on the disk: - _, ok := dependenciesOnDriveSet[replayHashExtension] + _, ok := dependenciesOnDriveSet[dependencyHashExtension] if ok { // the map is already downloaded, skip it: - log.WithField("map", replayHashExtension). - Debug("Map is already downloaded, continuing.") + log.WithField("dependency", dependencyHashExtension). + Debug("Dependency is already downloaded, continuing.") continue } toDownloadDependencyToFileMap[replayDependencyURL] = ReplayFilenameIsMap{ - DependencyFilename: replayHashExtension, + DependencyFilename: dependencyHashExtension, IsMap: replayProcessingDependencyInfo.IsMap, } } From a8e31c2e1d61e4687dff17722efd9b9ce820ba0e Mon Sep 17 00:00:00 2001 From: Kaszanas <34846245+Kaszanas@users.noreply.github.com> Date: Mon, 5 May 2025 02:14:43 +0200 Subject: [PATCH 8/9] Further logging improvements, adjusted error message --- dataproc/downloader/downloader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataproc/downloader/downloader.go b/dataproc/downloader/downloader.go index 7695c52..c76a46f 100644 --- a/dataproc/downloader/downloader.go +++ b/dataproc/downloader/downloader.go @@ -279,7 +279,7 @@ func downloadSingleDependency(taskState DownloadTaskState) { if err != nil { sendDownloadTaskReturnInfoToChannels( &taskState, - fmt.Errorf("error downloading in http.Get map: %v", err), + fmt.Errorf("error downloading in http.Get dependency: %v", err), ) return } From d68cedd22c965b0c108aa742ed7dd09832c5c5f4 Mon Sep 17 00:00:00 2001 From: Kaszanas <34846245+Kaszanas@users.noreply.github.com> Date: Mon, 5 May 2025 02:18:10 +0200 Subject: [PATCH 9/9] Adjusted citation information --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5f3b47b..90114e7 100644 --- a/README.md +++ b/README.md @@ -169,12 +169,12 @@ This repository is licensed under GNU GPL v3 license. If You would like to acqui author = {Białecki, Andrzej and Białecki, Piotr and Krupiński, Leszek}, - title = {{Kaszanas/SC2InfoExtractorGo: 1.2.0 + title = {{Kaszanas/SC2InfoExtractorGo: 2.1.3 SC2InfoExtractorGo Release}}, month = {jun}, year = {2022}, publisher = {Zenodo}, - version = {1.2.0}, + version = {2.1.3}, doi = {10.5281/zenodo.5296788}, url = {https://doi.org/10.5281/zenodo.5296788} }