Vipul cy011

ai-ml/Sample Records/cyber_threat_samples.json

@@ -0,0 +1,60 @@
+
+{
+  "threat_id": "b2e4d5f6-1111-4c3d-c111-000000000001",
+  "threat_type": "phishing",
+  "title": "Fake disaster relief payment email",
+  "description": "Email campaign impersonating government agencies offering emergency relief payments.",
+  "risk_level": "critical",
+  "status": "active",
+  "category": "cyber",
+  "confidence_score": 0.92,
+  "detected_at": "2026-03-21T12:10:00Z",
+  "source_id": "33333333-cccc-4ccc-dddd-000000000010",
+  "created_at": "2026-03-21T12:20:00Z",
+  "updated_at": "2026-03-21T12:20:00Z"
+}
+
+{
+  "threat_id": "b2e4d5f6-2222-4c3d-c222-000000000002",
+  "threat_type": "malware",
+  "title": "Malicious weather alert application",
+  "description": "Fake mobile application distributing malware under the guise of real-time weather alerts.",
+  "risk_level": "high",
+  "status": "active",
+  "category": "cyber",
+  "confidence_score": 0.87,
+  "detected_at": "2026-02-15T08:45:00Z",
+  "source_id": "33333333-cccc-4ccc-dddd-000000000011",
+  "created_at": "2026-02-15T09:00:00Z",
+  "updated_at": "2026-02-15T09:00:00Z"
+}
+
+{
+  "threat_id": "b2e4d5f6-3333-4c3d-c333-000000000003",
+  "threat_type": "misinformation",
+  "title": "False evacuation alert circulating online",
+  "description": "Social media posts spreading false evacuation notices during severe storm events.",
+  "risk_level": "medium",
+  "status": "monitoring",
+  "category": "cyber",
+  "confidence_score": 0.78,
+  "detected_at": "2026-01-30T16:20:00Z",
+  "source_id": "33333333-cccc-4ccc-dddd-000000000012",
+  "created_at": "2026-01-30T16:30:00Z",
+  "updated_at": "2026-01-30T16:30:00Z"
+}
+
+{
+  "threat_id": "b2e4d5f6-4444-4c3d-c444-000000000004",
+  "threat_type": "data_breach",
+  "title": "Unauthorized access to emergency response system",
+  "description": "Multiple unauthorized login attempts detected targeting emergency service infrastructure.",
+  "risk_level": "high",
+  "status": "active",
+  "category": "cyber",
+  "confidence_score": 0.85,
+  "detected_at": "2026-03-10T22:15:00Z",
+  "source_id": "33333333-cccc-4ccc-dddd-000000000013",
+  "created_at": "2026-03-10T22:30:00Z",
+  "updated_at": "2026-03-10T22:30:00Z"
+}

ai-ml/Sample Records/hazard_event_samples.json

@@ -0,0 +1,80 @@
+{
+  "hazard_event_id": "a1f3c2d4-1111-4a2b-b111-000000000001",
+  "hazard_type": "heatwave",
+  "description": "Extreme heatwave conditions affecting metropolitan communities.",
+  "severity_level": "high",
+  "event_status": "active",
+  "start_time": "2026-01-10T09:00:00Z",
+  "end_time": null,
+  "source_id": "33333333-cccc-4ccc-dddd-000000000001",
+  "source_ref_event": "BOM-HEAT-001",
+  "geo_location_id": "22222222-bbbb-4bbb-cccc-000000000001",
+  "created_at": "2026-01-10T09:15:00Z",
+  "updated_at": "2026-01-10T09:15:00Z"
+}
+
+{
+  "hazard_event_id": "a1f3c2d4-2222-4a2b-b222-000000000002",
+  "hazard_type": "storm",
+  "description": "Severe thunderstorm with heavy rainfall and strong winds.",
+  "severity_level": "medium",
+  "event_status": "active",
+  "start_time": "2026-02-05T14:30:00Z",
+  "end_time": null,
+  "source_id": "33333333-cccc-4ccc-dddd-000000000002",
+  "source_ref_event": "BOM-STORM-002",
+  "geo_location_id": "22222222-bbbb-4bbb-cccc-000000000002",
+  "created_at": "2026-02-05T14:45:00Z",
+  "updated_at": "2026-02-05T14:45:00Z"
+}
+
+{
+  "hazard_event_id": "a1f3c2d4-3333-4a2b-b333-000000000003",
+  "hazard_type": "bushfire",
+  "description": "Rapidly spreading bushfire threatening rural communities.",
+  "severity_level": "critical",
+  "event_status": "active",
+  "start_time": "2026-03-21T10:30:00Z",
+  "end_time": null,
+  "source_id": "33333333-cccc-4ccc-dddd-000000000003",
+  "source_ref_event": "BOM-FIRE-003",
+  "geo_location_id": "22222222-bbbb-4bbb-cccc-000000000003",
+  "created_at": "2026-03-21T11:05:00Z",
+  "updated_at": "2026-03-21T11:05:00Z"
+}
+
+{
+  "hazard_event_id": "a1f3c2d4-4444-4a2b-b444-000000000004",
+  "hazard_type": "flood",
+  "description": "River flooding caused by prolonged rainfall across low-lying areas.",
+  "severity_level": "high",
+  "event_status": "monitoring",
+  "start_time": "2026-04-01T06:00:00Z",
+  "end_time": null,
+  "source_id": "33333333-cccc-4ccc-dddd-000000000004",
+  "source_ref_event": "BOM-FLOOD-004",
+  "geo_location_id": "22222222-bbbb-4bbb-cccc-000000000004",
+  "created_at": "2026-04-01T06:15:00Z",
+  "updated_at": "2026-04-01T06:15:00Z"
+}
+
+{
+  "hazard_event_id": "a1f3c2d4-5555-4a2b-b555-000000000005",
+  "hazard_type": "cyclone",
+  "description": "Category 4 cyclone approaching coastal regions with destructive winds.",
+  "severity_level": "critical",
+  "event_status": "active",
+  "start_time": "2026-02-20T03:00:00Z",
+  "end_time": null,
+  "source_id": "33333333-cccc-4ccc-dddd-000000000005",
+  "source_ref_event": "BOM-CYC-005",
+  "geo_location_id": "22222222-bbbb-4bbb-cccc-000000000005",
+  "created_at": "2026-02-20T03:10:00Z",
+  "updated_at": "2026-02-20T03:10:00Z"
+}
+
+
+
+
+
+

ai-ml/Sample Records/risk_assessment_integration.json

@@ -0,0 +1,64 @@
+
+{
+  "integration_event_id": "c3f5e6a7-1111-4d4e-d111-000000000001",
+  "related_hazard_event_id": "a1f3c2d4-3333-4a2b-b333-000000000003",
+  "related_threat_id": "b2e4d5f6-1111-4c3d-c111-000000000001",
+  "correlation_score": 0.88,
+  "linkage_reason": "Phishing campaign detected during active bushfire emergency targeting affected communities.",
+  "integration_confidence": 0.91,
+  "linked_event_type": 3,
+  "event_status": 1,
+  "event_time": "2026-03-21T12:00:00Z",
+  "detected_at": "2026-03-21T12:10:00Z",
+  "reported_at": "2026-03-21T12:20:00Z",
+  "created_at": "2026-03-21T12:25:00Z",
+  "updated_at": "2026-03-21T12:25:00Z"
+}
+
+{
+  "integration_event_id": "c3f5e6a7-2222-4d4e-d222-000000000002",
+  "related_hazard_event_id": "a1f3c2d4-2222-4a2b-b222-000000000002",
+  "related_threat_id": "b2e4d5f6-3333-4c3d-c333-000000000003",
+  "correlation_score": 0.72,
+  "linkage_reason": "False evacuation alerts spread during storm warnings causing public confusion.",
+  "integration_confidence": 0.80,
+  "linked_event_type": 3,
+  "event_status": 2,
+  "event_time": "2026-02-05T15:00:00Z",
+  "detected_at": "2026-02-05T15:10:00Z",
+  "reported_at": "2026-02-05T15:25:00Z",
+  "created_at": "2026-02-05T15:30:00Z",
+  "updated_at": "2026-02-05T15:30:00Z"
+}
+
+{
+  "integration_event_id": "c3f5e6a7-3333-4d4e-d333-000000000003",
+  "related_hazard_event_id": "a1f3c2d4-4444-4a2b-b444-000000000004",
+  "related_threat_id": null,
+  "correlation_score": 0.60,
+  "linkage_reason": "Flood event monitored without associated cyber threat.",
+  "integration_confidence": 0.70,
+  "linked_event_type": 1,
+  "event_status": 2,
+  "event_time": "2026-04-01T07:00:00Z",
+  "detected_at": "2026-04-01T07:10:00Z",
+  "reported_at": "2026-04-01T07:20:00Z",
+  "created_at": "2026-04-01T07:25:00Z",
+  "updated_at": "2026-04-01T07:25:00Z"
+}
+
+{
+  "integration_event_id": "c3f5e6a7-4444-4d4e-d444-000000000004",
+  "related_hazard_event_id": null,
+  "related_threat_id": "b2e4d5f6-2222-4c3d-c222-000000000002",
+  "correlation_score": 0.65,
+  "linkage_reason": "Malware campaign detected independently of hazard events.",
+  "integration_confidence": 0.75,
+  "linked_event_type": 2,
+  "event_status": 1,
+  "event_time": "2026-02-15T09:00:00Z",
+  "detected_at": "2026-02-15T09:10:00Z",
+  "reported_at": "2026-02-15T09:20:00Z",
+  "created_at": "2026-02-15T09:25:00Z",
+  "updated_at": "2026-02-15T09:25:00Z"
+}

ai-ml/cleaning/README.md

@@ -0,0 +1,134 @@
+# Data Cleaning Pipeline (`ai-ml/cleaning`)
+
+This module provides a configurable CSV cleaning + validation pipeline.
+
+It is designed to:
+
+- clean raw tabular data (missing values, duplicates, type conversion, string normalization)
+- validate cleaned data against rules (required fields, allowed values, ranges, types, date formats)
+- generate output artifacts (cleaned CSV, validation report, comparison report, pipeline log)
+
+## Folder Structure
+
+- `config/pipeline_config.json` - all pipeline configuration
+- `data/input/` - raw CSV input files
+- `data/output/` - cleaned CSV output
+- `data/reports/` - JSON reports (validation + before/after comparison)
+- `data/logs/` - pipeline event log
+- `src/` - pipeline implementation
+
+## What The Pipeline Does
+
+For each run, the pipeline:
+
+1. reads input CSV from `paths.input_csv`
+2. selects cleaning/validation rules:
+
+- dataset-specific rules under `datasets.<name>` when columns match
+- otherwise falls back to top-level `cleaning` + `validation` (`generic`)
+
+1. applies cleaning steps
+2. runs validation checks
+3. writes outputs to configured paths
+
+## Run The Pipeline
+
+From repo root:
+
+```powershell
+python ai-ml/cleaning/src/main.py
+```
+
+With a custom config:
+
+```powershell
+python ai-ml/cleaning/src/main.py --config ai-ml/cleaning/config/pipeline_config.json
+```
+
+## Use In Other Python Scripts
+
+Example integration:
+
+```python
+from pathlib import Path
+import sys
+
+cleaning_root = Path("ai-ml/cleaning").resolve()
+if str(cleaning_root) not in sys.path:
+    sys.path.insert(0, str(cleaning_root))
+
+from src.pipeline import run_pipeline
+
+summary = run_pipeline(cleaning_root / "config" / "pipeline_config.json")
+print(summary)
+```
+
+## How To Modify `pipeline_config.json`
+
+### 1. Set input/output paths
+
+Update `paths`:
+
+- `input_csv`
+- `cleaned_csv`
+- `validation_report`
+- `comparison_report`
+- `pipeline_log`
+
+### 2. Configure generic fallback rules
+
+Top-level `cleaning` and `validation` are used when no dataset-specific schema matches.
+
+### 3. Add or edit dataset-specific rules
+
+Under `datasets`, each dataset entry should contain:
+
+- `cleaning`
+- `validation.required_columns`
+- `validation.column_rules`
+
+Minimal pattern:
+
+```json
+"datasets": {
+  "my_dataset": {
+    "cleaning": {
+      "missing_values": { "drop": [], "fill": {} },
+      "duplicates": { "subset": [] },
+      "type_conversion": { "int": [], "float": [], "datetime": [] },
+      "string_standardisation": ["col_a", "col_b"]
+    },
+    "validation": {
+      "required_columns": ["id"],
+      "column_rules": {
+        "id": { "required": true, "type": "int", "unique": true }
+      }
+    }
+  }
+}
+```
+
+## Validation Rules Supported
+
+Per column (`validation.column_rules.<column>`):
+
+- `required: true|false`
+- `type: "int" | "float" | "str" | "date"`
+- `unique: true|false`
+- `allowed_values: [...]`
+- `min` / `max` (numeric)
+- `format` (for `type: "date"`)
+
+## Cleaning Rules Supported
+
+- `missing_values.drop`: drop rows where these columns are null
+- `missing_values.fill`: fill nulls with provided values
+- `duplicates.subset`: drop duplicate rows by subset
+- `type_conversion.int|float|datetime`: coercive conversion
+- `string_standardisation`: trim + normalize configured text columns
+
+## Notes
+
+- Input is expected to be CSV.
+- Validation `FAIL` means rule violations were found; pipeline still completes and writes reports.
+- If a new dataset is not being picked, verify its `required_columns` match the CSV column names exactly.