-
Notifications
You must be signed in to change notification settings - Fork 0
#6 - Hop Dataset Import #31
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
17 commits
Select commit
Hold shift + click to select a range
79e8106
Add command to import hop varieties
krzysztofkozyra021 8f3f0fa
Move test JSON5 files to fixtures
krzysztofkozyra021 66f3739
Refactor, change approach to fake storage
krzysztofkozyra021 c79092b
Merge branch 'main' into hop-dataset-import
krzysztofkozyra021 2f2c8ca
feat(hops): expand schema with agronomic, lineage, thiols and alt_nam…
krzysztofkozyra021 34a94cd
feat(hops): map and validate all new hop fields in import pipeline
krzysztofkozyra021 a6acc44
test(hops): update tests and factory for new schema
krzysztofkozyra021 cb71a4b
Remove unnecessary config
krzysztofkozyra021 8f82c40
remove test files leftovers
krzysztofkozyra021 42277e1
Add parameter to specify folder for import
krzysztofkozyra021 448aaf3
chore: ignore storage/framework/testing/disks from git
krzysztofkozyra021 7c96d58
Apply suggestions
krzysztofkozyra021 1559c53
Merge branch 'main' into hop-dataset-import
krzysztofkozyra021 cda96af
Apply suggestions
krzysztofkozyra021 ad063a8
Merge branch 'main' into hop-dataset-import
krzysztofkozyra021 bb00412
Add Enums
krzysztofkozyra021 b852260
Update Hop Model for Enums
krzysztofkozyra021 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,3 +22,4 @@ supervisord.pid | |
| .php-cs-fixer.cache | ||
| phpunit.xml | ||
| /database/sqlite | ||
| /storage/framework/testing/disks/ | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace HopsWeb\Actions; | ||
|
|
||
| use HopsWeb\ValueObjects\RangeOrNumber; | ||
|
|
||
| class MapHopData | ||
| { | ||
| public function execute(array $data): array | ||
| { | ||
| $ingredients = $data["ingredients"] ?? []; | ||
| $agronomic = $data["agronomic"] ?? []; | ||
| $yield = $agronomic["yield"] ?? null; | ||
|
|
||
| return [ | ||
| "name" => $data["name"] ?? null, | ||
| "alt_name" => $data["altName"] ?? null, | ||
| "country" => $data["country"] ?? null, | ||
| "description" => $data["origin"] ?? null, | ||
| "descriptors" => $data["descriptors"] ?? [], | ||
| "lineage" => $data["lineage"] ?? [], | ||
| "alpha_acid" => $this->extractRange($ingredients["alphas"] ?? null), | ||
| "beta_acid" => $this->extractRange($ingredients["betas"] ?? null), | ||
| "cohumulone" => $this->extractRange($ingredients["cohumulones"] ?? null), | ||
| "total_oil" => $this->extractRange($ingredients["oils"] ?? null), | ||
| "polyphenol" => $this->extractRange($ingredients["polyphenols"] ?? null), | ||
| "xanthohumol" => $this->extractRange($ingredients["xanthohumols"] ?? null), | ||
| "farnesene" => $this->extractRange($ingredients["farnesenes"] ?? null), | ||
| "linalool" => $this->extractRange($ingredients["linalool"] ?? null), | ||
| "thiols" => $ingredients["thiols"] ?? null, | ||
| "aroma_citrusy" => $data["aroma"]["citrusy"] ?? null, | ||
| "aroma_fruity" => $data["aroma"]["fruity"] ?? null, | ||
| "aroma_floral" => $data["aroma"]["floral"] ?? null, | ||
| "aroma_herbal" => $data["aroma"]["herbal"] ?? null, | ||
| "aroma_spicy" => $data["aroma"]["spicy"] ?? null, | ||
| "aroma_resinous" => $data["aroma"]["resinous"] ?? null, | ||
| "aroma_sugarlike" => $data["aroma"]["sugarlike"] ?? null, | ||
| "aroma_misc" => $data["aroma"]["misc"] ?? null, | ||
| "aroma_descriptors" => $data["aromaDescription"] ?? [], | ||
| "yield_min" => is_array($yield) ? ($yield["min"] ?? null) : null, | ||
| "yield_max" => is_array($yield) ? ($yield["max"] ?? null) : null, | ||
| "maturity" => $agronomic["maturity"] ?? null, | ||
| "wilt_disease" => $agronomic["wiltDisease"] ?? null, | ||
| "downy_mildew" => $agronomic["downyMildew"] ?? null, | ||
| "powdery_mildew" => $agronomic["powderyMildew"] ?? null, | ||
| "aphid" => $agronomic["aphid"] ?? null, | ||
| "substitutes" => $this->extractSubstitutes($data), | ||
| ]; | ||
| } | ||
|
|
||
| private function extractRange(?array $ingredientData): ?RangeOrNumber | ||
| { | ||
| if ($ingredientData === null) { | ||
| return null; | ||
| } | ||
|
|
||
| $min = $ingredientData["min"] ?? null; | ||
| $max = $ingredientData["max"] ?? null; | ||
|
|
||
| if ($min === null && $max === null) { | ||
| return null; | ||
| } | ||
|
|
||
| if ($min !== null && $max !== null) { | ||
| return RangeOrNumber::fromRange((float)$min, (float)$max); | ||
| } | ||
|
|
||
| return RangeOrNumber::fromNumber((float)($min ?? $max)); | ||
| } | ||
|
|
||
| private function extractSubstitutes(array $data): array | ||
| { | ||
| $alternatives = $data["ingredients"]["alternatives"] ?? []; | ||
|
|
||
| return [ | ||
| "brewhouse" => $alternatives["brewhouse"] ?? [], | ||
| "dryhopping" => $alternatives["dryhopping"] ?? [], | ||
| ]; | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,139 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace HopsWeb\Actions; | ||
|
|
||
| use HopsWeb\Models\Hop; | ||
| use HopsWeb\ValueObjects\RangeOrNumber; | ||
| use Illuminate\Support\Facades\Validator; | ||
| use Illuminate\Support\Str; | ||
| use Illuminate\Validation\ValidationException; | ||
|
|
||
| class UpsertHop | ||
| { | ||
| public function execute(array $data): Hop | ||
| { | ||
| $validated = $this->validate($data); | ||
|
|
||
| return Hop::updateOrCreate( | ||
| ["name" => $validated["name"]], | ||
| $this->buildAttributes($validated), | ||
| ); | ||
| } | ||
|
|
||
| private function validate(array $data): array | ||
| { | ||
| $this->validateRangeFields($data); | ||
|
|
||
| $filteredData = array_diff_key($data, array_flip(Hop::RANGE_FIELDS)); | ||
|
|
||
| $validator = Validator::make($filteredData, [ | ||
| "name" => ["required", "string", "max:255"], | ||
| "alt_name" => ["nullable", "string", "max:255"], | ||
| "country" => ["nullable", "string", "max:255"], | ||
| "description" => ["nullable", "string"], | ||
| "descriptors" => ["nullable", "array"], | ||
| "descriptors.*" => ["string"], | ||
| "lineage" => ["nullable", "array"], | ||
| "lineage.*" => ["string"], | ||
| "thiols" => ["nullable", "string", "in:low,medium,high"], | ||
| "aroma_citrusy" => ["nullable", "integer", "between:0,5"], | ||
| "aroma_fruity" => ["nullable", "integer", "between:0,5"], | ||
| "aroma_floral" => ["nullable", "integer", "between:0,5"], | ||
| "aroma_herbal" => ["nullable", "integer", "between:0,5"], | ||
| "aroma_spicy" => ["nullable", "integer", "between:0,5"], | ||
| "aroma_resinous" => ["nullable", "integer", "between:0,5"], | ||
| "aroma_sugarlike" => ["nullable", "integer", "between:0,5"], | ||
| "aroma_misc" => ["nullable", "integer", "between:0,5"], | ||
| "aroma_descriptors" => ["nullable", "array"], | ||
| "aroma_descriptors.*" => ["string"], | ||
| "substitutes" => ["nullable", "array"], | ||
| "substitutes.brewhouse" => ["nullable", "array"], | ||
| "substitutes.brewhouse.*" => ["string"], | ||
| "substitutes.dryhopping" => ["nullable", "array"], | ||
| "substitutes.dryhopping.*" => ["string"], | ||
| "yield_min" => ["nullable", "integer", "min:0"], | ||
| "yield_max" => ["nullable", "integer", "min:0"], | ||
| "maturity" => ["nullable", "string", "max:255"], | ||
| "wilt_disease" => ["nullable", "string", "max:255"], | ||
| "downy_mildew" => ["nullable", "string", "max:255"], | ||
| "powdery_mildew" => ["nullable", "string", "max:255"], | ||
| "aphid" => ["nullable", "string", "max:255"], | ||
| ]); | ||
|
|
||
| if ($validator->fails()) { | ||
| throw new ValidationException($validator); | ||
| } | ||
|
|
||
| $validated = $validator->validated(); | ||
|
|
||
| foreach (Hop::RANGE_FIELDS as $field) { | ||
| $validated[$field] = $data[$field] ?? null; | ||
| } | ||
|
|
||
| return $validated; | ||
| } | ||
|
|
||
| private function validateRangeFields(array $data): void | ||
| { | ||
| $errors = []; | ||
|
|
||
| foreach (Hop::RANGE_FIELDS as $field) { | ||
| $value = $data[$field] ?? null; | ||
|
|
||
| if ($value === null) { | ||
| continue; | ||
| } | ||
|
|
||
| if (!$value instanceof RangeOrNumber) { | ||
| $errors[$field][] = "The {$field} field must be a valid RangeOrNumber instance."; | ||
| } | ||
| } | ||
|
|
||
| if (!empty($errors)) { | ||
| throw ValidationException::withMessages($errors); | ||
| } | ||
| } | ||
|
|
||
| private function buildAttributes(array $validated): array | ||
| { | ||
| return [ | ||
| "slug" => Str::slug($validated["name"]) . "-hop", | ||
| "alt_name" => $validated["alt_name"] ?? null, | ||
| "country" => $validated["country"] ?? null, | ||
| "description" => $validated["description"] ?? null, | ||
| "descriptors" => $validated["descriptors"] ?? [], | ||
| "lineage" => $validated["lineage"] ?? [], | ||
| "alpha_acid" => $validated["alpha_acid"], | ||
| "beta_acid" => $validated["beta_acid"], | ||
| "cohumulone" => $validated["cohumulone"], | ||
| "total_oil" => $validated["total_oil"], | ||
| "polyphenol" => $validated["polyphenol"], | ||
| "xanthohumol" => $validated["xanthohumol"], | ||
| "farnesene" => $validated["farnesene"], | ||
| "linalool" => $validated["linalool"], | ||
| "thiols" => $validated["thiols"] ?? null, | ||
| "aroma_citrusy" => $validated["aroma_citrusy"] ?? null, | ||
| "aroma_fruity" => $validated["aroma_fruity"] ?? null, | ||
| "aroma_floral" => $validated["aroma_floral"] ?? null, | ||
| "aroma_herbal" => $validated["aroma_herbal"] ?? null, | ||
| "aroma_spicy" => $validated["aroma_spicy"] ?? null, | ||
| "aroma_resinous" => $validated["aroma_resinous"] ?? null, | ||
| "aroma_sugarlike" => $validated["aroma_sugarlike"] ?? null, | ||
| "aroma_misc" => $validated["aroma_misc"] ?? null, | ||
| "aroma_descriptors" => $validated["aroma_descriptors"] ?? [], | ||
| "substitutes" => [ | ||
| "brewhouse" => $validated["substitutes"]["brewhouse"] ?? [], | ||
| "dryhopping" => $validated["substitutes"]["dryhopping"] ?? [], | ||
| ], | ||
krzysztofkozyra021 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| "yield_min" => $validated["yield_min"] ?? null, | ||
| "yield_max" => $validated["yield_max"] ?? null, | ||
| "maturity" => $validated["maturity"] ?? null, | ||
| "wilt_disease" => $validated["wilt_disease"] ?? null, | ||
| "downy_mildew" => $validated["downy_mildew"] ?? null, | ||
| "powdery_mildew" => $validated["powdery_mildew"] ?? null, | ||
| "aphid" => $validated["aphid"] ?? null, | ||
| ]; | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace HopsWeb\Console\Commands; | ||
|
|
||
| use HopsWeb\Jobs\ImportHopVarietyJob; | ||
| use Illuminate\Console\Command; | ||
| use Illuminate\Filesystem\FilesystemManager; | ||
|
|
||
| class HopsImportCommand extends Command | ||
| { | ||
| protected $signature = "hops:import {folder=hops_data : Directory inside storage to scan for JSON/JSON5 files}"; | ||
| protected $description = "Import hop varieties from JSON/JSON5 files in storage"; | ||
|
|
||
| public function handle(FilesystemManager $filesystem): void | ||
| { | ||
| $folder = $this->argument("folder"); | ||
|
|
||
krzysztofkozyra021 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| $files = collect($filesystem->files($folder)) | ||
| ->filter(fn(string $file): bool => in_array( | ||
| pathinfo($file, PATHINFO_EXTENSION), | ||
| ["json", "json5"], | ||
| true, | ||
| )); | ||
|
|
||
| if ($files->isEmpty()) { | ||
| $this->info("No files found in {$folder} directory."); | ||
|
|
||
| return; | ||
| } | ||
|
|
||
| $this->info("Found {$files->count()} hop variety file(s) to import."); | ||
|
|
||
| foreach ($files as $file) { | ||
| $this->info("Dispatching import for: {$file}"); | ||
| ImportHopVarietyJob::dispatch($file); | ||
| } | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace HopsWeb\Enums; | ||
|
|
||
| use HopsWeb\Traits\EnumValues; | ||
|
|
||
| enum HopDescriptor: string | ||
| { | ||
| use EnumValues; | ||
|
|
||
| case Fruity = "fruity"; | ||
| case Citrusy = "citrusy"; | ||
| case Herbal = "herbal"; | ||
| case Spicy = "spicy"; | ||
| case Floral = "floral"; | ||
| case Resinous = "resinous"; | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace HopsWeb\Enums; | ||
|
|
||
| use HopsWeb\Traits\EnumValues; | ||
|
|
||
| enum HopLineage: string | ||
| { | ||
| use EnumValues; | ||
|
|
||
| case BrewersGold = "brewers-gold"; | ||
| case Fuggle = "fuggle"; | ||
| case Cascade = "cascade"; | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace HopsWeb\Enums; | ||
|
|
||
| use HopsWeb\Traits\EnumValues; | ||
|
|
||
| enum HopMaturity: string | ||
| { | ||
| use EnumValues; | ||
|
|
||
| case Early = "early"; | ||
| case MidEarly = "mid early"; | ||
| case MidLate = "mid late"; | ||
| case Late = "late"; | ||
| case VeryLate = "very late"; | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace HopsWeb\Enums; | ||
|
|
||
| use HopsWeb\Traits\EnumValues; | ||
|
|
||
| enum Resistance: string | ||
| { | ||
| use EnumValues; | ||
|
|
||
| case Resistant = "resistant"; | ||
| case Tolerant = "tolerant"; | ||
| case Susceptible = "susceptible"; | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| <?php | ||
|
|
||
| declare(strict_types=1); | ||
|
|
||
| namespace HopsWeb\Helpers; | ||
|
|
||
| class Json5Parser | ||
| { | ||
kamilpiech97 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| public function parse(string $content): ?array | ||
| { | ||
krzysztofkozyra021 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| $json = preg_replace('/(?<=[{,\n])\s*([a-zA-Z_]\w*)\s*:/m', '"$1":', $content); | ||
|
|
||
| $data = json_decode($json, true); | ||
|
|
||
krzysztofkozyra021 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| return is_array($data) ? $data : null; | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.