From a9e359bf60b165e129ea21cf583122a812db00d4 Mon Sep 17 00:00:00 2001 From: Oleg Ozimok Date: Thu, 19 Feb 2026 15:08:15 +0200 Subject: [PATCH] add GeoLite2-ASN support and advanced filtering --- .gitignore | 4 +- README.md | 39 ++++++++++++++--- mmdb2csv.go | 122 ++++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 139 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 66fd13c..50dcd6a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,5 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out -# Dependency directories (remove the comment below to include it) -# vendor/ +# ide +/.idea diff --git a/README.md b/README.md index e9faa88..3bf7c6b 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,43 @@ Convert Maxmind mmdb database to CSV. -# Why? +# Supported Databases -Many applications support CSV but not mmdb. For example it's easy to import CSV to SQL databases. +Automatically detects database type based on filename: +- City +- Country +- Connections +- ISP +- ASN **(New!)** -# How? +# Advanced Filtering -./mmdb2csv GeoIP2ISP.mmdb > isp.csv +You can filter the output by any field value using the following flags: +- `-filter-field`: The name of the field to filter by (e.g., `autonomous_system_organization` or `prefix`). +- `-filter-value`: The value to search for. + +## Filtering Features +- **Substring Match**: Filters match any part of the field value (case-insensitive). +- **IP Lookup**: If filtering by the `prefix` field with a specific IP address, the tool finds the CIDR range that contains that IP. + +# Examples + +## Basic conversion +```bash +./mmdb2csv GeoLite2-ASN.mmdb > asn.csv +``` + +## Filter by organization (substring) +```bash +./mmdb2csv -filter-field autonomous_system_organization -filter-value "Opera Norway" GeoLite2-ASN.mmdb +``` + +## IP address lookup +```bash +./mmdb2csv -filter-field prefix -filter-value "82.145.223.76" GeoLite2-ASN.mmdb +``` # Build +```bash go build mmdb2csv.go - +``` diff --git a/mmdb2csv.go b/mmdb2csv.go index 4ca0684..e8e9de5 100644 --- a/mmdb2csv.go +++ b/mmdb2csv.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "log" + "net" "os" "path" "strings" @@ -15,6 +16,8 @@ import ( // ClickHouse CSV mode var clickhouseFlag bool +var filterField string +var filterValue string func removeUnsafeChars(strarr []string) []string { var output = []string{} @@ -30,6 +33,34 @@ func containsIgnoreCase(s string, substr string) bool { return strings.Contains(strings.ToLower(s), strings.ToLower(substr)) } +func shouldInclude(headers []string, values []string) bool { + if filterField == "" { + return true + } + + for i, header := range headers { + if header == filterField { + if i < len(values) { + val := values[i] + // Special case for prefix filtering with an IP + if header == "prefix" { + ip := net.ParseIP(filterValue) + if ip != nil { + _, cidrNet, err := net.ParseCIDR(val) + if err == nil { + return cidrNet.Contains(ip) + } + } + } + // Default to substring match + return strings.Contains(strings.ToLower(val), strings.ToLower(filterValue)) + } + return false + } + } + return false +} + func dumpCity(networks *maxminddb.Networks, writer *csv.Writer) (err error) { headers := []string{ "prefix", @@ -132,13 +163,16 @@ func dumpCity(networks *maxminddb.Networks, writer *csv.Writer) (err error) { fmt.Sprintf("%v", record.Traits.IsAnonymousProxy), fmt.Sprintf("%v", record.Traits.IsSatelliteProvider), ) + if clickhouseFlag { - err = writer.Write(removeUnsafeChars(values)) - } else { - err = writer.Write(values) + values = removeUnsafeChars(values) } - if err != nil { - return err + + if shouldInclude(headers, values) { + err = writer.Write(values) + if err != nil { + return err + } } } return nil @@ -164,13 +198,16 @@ func dumpConnections(networks *maxminddb.Networks, writer *csv.Writer) (err erro subnet.String(), record.ConnectionType, } + if clickhouseFlag { - err = writer.Write(removeUnsafeChars(values)) - } else { - err = writer.Write(values) + values = removeUnsafeChars(values) } - if err != nil { - return err + + if shouldInclude(headers, values) { + err = writer.Write(values) + if err != nil { + return err + } } } return nil @@ -235,13 +272,16 @@ func dumpCountry(networks *maxminddb.Networks, writer *csv.Writer) (err error) { fmt.Sprintf("%v", record.Traits.IsAnonymousProxy), fmt.Sprintf("%v", record.Traits.IsSatelliteProvider), } + if clickhouseFlag { - err = writer.Write(removeUnsafeChars(values)) - } else { - err = writer.Write(values) + values = removeUnsafeChars(values) } - if err != nil { - return err + + if shouldInclude(headers, values) { + err = writer.Write(values) + if err != nil { + return err + } } } return nil @@ -273,13 +313,53 @@ func dumpISP(networks *maxminddb.Networks, writer *csv.Writer) (err error) { record.ISP, record.Organization, } + if clickhouseFlag { - err = writer.Write(removeUnsafeChars(values)) - } else { + values = removeUnsafeChars(values) + } + + if shouldInclude(headers, values) { err = writer.Write(values) + if err != nil { + return err + } } + } + return nil +} + +func dumpASN(networks *maxminddb.Networks, writer *csv.Writer) (err error) { + headers := []string{ + "prefix", + "autonomous_system_number", + "autonomous_system_organization", + } + err = writer.Write(headers) + if err != nil { + return err + } + + record := geoip2.ASN{} + for networks.Next() { + subnet, err := networks.Network(&record) if err != nil { - return err + log.Fatalln(err) + } + values := []string{ + subnet.String(), + fmt.Sprintf("%d", record.AutonomousSystemNumber), + record.AutonomousSystemOrganization, + } + + if clickhouseFlag { + values = removeUnsafeChars(values) + } + + if shouldInclude(headers, values) { + err = writer.Write(values) + if err != nil { + return err + } } } return nil @@ -287,6 +367,8 @@ func dumpISP(networks *maxminddb.Networks, writer *csv.Writer) (err error) { func main() { flag.BoolVar(&clickhouseFlag, "c", false, "for ClickHouse dictionary") + flag.StringVar(&filterField, "filter-field", "", "field to filter by") + flag.StringVar(&filterValue, "filter-value", "", "value to filter by") flag.Parse() if flag.NArg() == 0 { @@ -326,8 +408,10 @@ func main() { err2 = dumpCountry(networks, writer) } else if containsIgnoreCase(fname, "isp") { err2 = dumpISP(networks, writer) + } else if containsIgnoreCase(fname, "asn") { + err2 = dumpASN(networks, writer) } else { - log.Fatal("Dump type not recognized, please rename mmdb file to contain any of the following strings: city, connections, country, or isp") + log.Fatal("Dump type not recognized, please rename mmdb file to contain any of the following strings: city, connections, country, isp, or asn") } if err2 != nil { log.Fatal(err2.Error())